From 5f53c7dc623b5dd2b984ed952be1c328efed9f75 Mon Sep 17 00:00:00 2001 From: tis24dev Date: Sun, 2 Aug 2026 22:06:20 +0200 Subject: [PATCH 01/50] refactor(restore): single plan body builder shared by CLI and TUI The restore plan was rendered twice: ShowRestorePlan(logger, config) in selective.go printed it for the CLI while buildRestorePlanText(config) built it for the Charm front-end. The two had already drifted -- the CLI switch had no default arm for an unknown RestoreMode, its TFA/WebAuthn advisory was a single 140-column line with no separating blank line, and a nil config made it panic on config.Mode. Move buildRestorePlanText into its own neutral file so the shared body is not filed under a TUI entrypoint (which is how the CLI copy drifted in the first place), and reduce both front-ends to chrome: the CLI keeps its ASCII banner, the Charm side keeps its styled Pager title. The CLI now renders the TUI's wrapped two-line TFA advisory and gains the "Unknown mode" default arm. fmt.Print, not Printf: category paths reach the body verbatim and a '%' in a path would be consumed as a format verb. cliWorkflowUI.ShowRestorePlan now returns an error for a nil config instead of dereferencing it, matching the Charm sibling; a test pins it, as one already did on the Charm side. Output stays on os.Stdout. The CLI restore prompts are already split across stdout and the injected writer, so moving only the plan would not unify anything; that is deferred to its own change. --- docs/RESTORE_TECHNICAL.md | 14 ++-- .../orchestrator/additional_helpers_test.go | 8 +- internal/orchestrator/restore_plan_render.go | 74 +++++++++++++++++++ .../orchestrator/restore_plan_render_test.go | 68 +++++++++++++++++ internal/orchestrator/restore_tui.go | 59 --------------- internal/orchestrator/restore_tui_test.go | 63 ---------------- internal/orchestrator/selective.go | 63 ---------------- internal/orchestrator/workflow_ui_cli.go | 24 +++++- internal/orchestrator/workflow_ui_cli_test.go | 11 +++ 9 files changed, 192 insertions(+), 192 deletions(-) create mode 100644 internal/orchestrator/restore_plan_render.go create mode 100644 internal/orchestrator/restore_plan_render_test.go diff --git a/docs/RESTORE_TECHNICAL.md b/docs/RESTORE_TECHNICAL.md index 5a6345db..559a66eb 100644 --- a/docs/RESTORE_TECHNICAL.md +++ b/docs/RESTORE_TECHNICAL.md @@ -130,7 +130,8 @@ Completion Summary | `cmd/proxsave/main.go` | Entry point, CLI parsing | `main()`, flag handling | | `internal/orchestrator/restore.go` | Entry stub (body in `restore_workflow_ui_run.go`) | `RunRestoreWorkflow()` | | `internal/orchestrator/categories.go` | Category definitions | `GetAllCategories()`, `PathMatchesCategory()` | -| `internal/orchestrator/selective.go` | Category selection/plan UI | `ShowRestoreModeMenuWithReader()`, `ShowRestorePlan()` | +| `internal/orchestrator/selective.go` | Category selection UI | `ShowRestoreModeMenuWithReader()`, `ShowCategorySelectionMenuWithReader()` | +| `internal/orchestrator/restore_plan_render.go` | Shared restore-plan body (no chrome) | `buildRestorePlanText()` | | `internal/orchestrator/decrypt.go` | Decryption workflow | `prepareDecryptedBackup()` | | `internal/orchestrator/compatibility.go` | System validation | `ValidateCompatibility()` | | `internal/orchestrator/backup_safety.go` | Safety backups | `CreateSafetyBackup()` | @@ -326,10 +327,13 @@ type Category struct { - Commands: number, 'a', 'n', 'c', 'b', '0' - Reached via the `RestoreWorkflowUI.SelectCategories()` interface -3. **`ShowRestorePlan()`** (`selective.go`): - - Display selected categories - - Show file paths to be restored - - Display warnings +3. **`buildRestorePlanText()`** (`restore_plan_render.go`): + - Renders the plan *body* only: selected categories, file paths to be restored, warnings + - Chrome is owned by the front-ends: `cliWorkflowUI.ShowRestorePlan()` + (`workflow_ui_cli.go`) prints the ASCII "RESTORE PLAN" banner around it, while + `charmWorkflowUI.ShowRestorePlan()` (`workflow_ui_charm_restore.go`) feeds it to a Pager + - Every body line must fit 80 columns (the Pager does not wrap); + `TestBuildRestorePlanTextLinesFit80Columns` pins this - Reached via the `RestoreWorkflowUI.ShowRestorePlan()` interface 4. **`ConfirmRestoreOperationWithReader()`** (`selective.go`): diff --git a/internal/orchestrator/additional_helpers_test.go b/internal/orchestrator/additional_helpers_test.go index 8c2fac62..e72da979 100644 --- a/internal/orchestrator/additional_helpers_test.go +++ b/internal/orchestrator/additional_helpers_test.go @@ -1079,6 +1079,8 @@ func TestShowRestorePlanOutputsPaths(t *testing.T) { SystemType: SystemTypePVE, } + ui := newCLIWorkflowUI(nil, logger) + var out bytes.Buffer old := os.Stdout r, w, _ := os.Pipe() @@ -1089,12 +1091,16 @@ func TestShowRestorePlanOutputsPaths(t *testing.T) { close(done) }() - ShowRestorePlan(logger, cfg) + planErr := ui.ShowRestorePlan(context.Background(), cfg) _ = w.Close() os.Stdout = old <-done + if planErr != nil { + t.Fatalf("ShowRestorePlan error: %v", planErr) + } + output := out.String() if !strings.Contains(output, "RESTORE PLAN") || !strings.Contains(output, "/etc/network") { t.Fatalf("unexpected plan output: %s", output) diff --git a/internal/orchestrator/restore_plan_render.go b/internal/orchestrator/restore_plan_render.go new file mode 100644 index 00000000..29197562 --- /dev/null +++ b/internal/orchestrator/restore_plan_render.go @@ -0,0 +1,74 @@ +package orchestrator + +import ( + "fmt" + "sort" + "strings" +) + +// buildRestorePlanText renders the restore plan *body* shared by both +// front-ends: the Charm UI feeds it to a Pager, and the CLI prints it under its +// own ASCII banner. Chrome (banners, titles, framing) belongs to the UIs, not +// here, so the two never drift on the content itself. +// +// Every line must fit 80 columns: the Pager does not wrap, so an over-width line +// is truncated and only reachable via horizontal scroll. +// TestBuildRestorePlanTextLinesFit80Columns pins this. +func buildRestorePlanText(config *SelectiveRestoreConfig) string { + if config == nil { + return "" + } + + var b strings.Builder + + // No ASCII banner: the Pager renders the styled "Restore plan" title, and the + // legacy box rule was cosmetically inconsistent with the Charm screens. + modeName := "" + switch config.Mode { + case RestoreModeFull: + modeName = "FULL restore (all categories)" + case RestoreModeStorage: + if config.SystemType.SupportsPVE() && !config.SystemType.SupportsPBS() { + modeName = "STORAGE only (cluster + storage + jobs + mounts)" + } else if config.SystemType.SupportsPBS() && !config.SystemType.SupportsPVE() { + modeName = "DATASTORE only (datastores + jobs + mounts)" + } else { + modeName = "STORAGE/DATASTORE only (PVE + PBS storage/jobs + mounts)" + } + case RestoreModeBase: + modeName = "SYSTEM BASE only (network + SSL + SSH + services + filesystem)" + case RestoreModeCustom: + modeName = fmt.Sprintf("CUSTOM selection (%d categories)", len(config.SelectedCategories)) + default: + modeName = "Unknown mode" + } + + fmt.Fprintf(&b, "Restore mode: %s\n", modeName) + fmt.Fprintf(&b, "System type: %s\n\n", GetSystemTypeString(config.SystemType)) + + b.WriteString("Categories to restore:\n") + for i, cat := range config.SelectedCategories { + fmt.Fprintf(&b, " %d. %s\n", i+1, cat.Name) + fmt.Fprintf(&b, " %s\n", cat.Description) + } + + b.WriteString("\nFiles/directories that will be restored:\n") + allPaths := GetSelectedPaths(config.SelectedCategories) + sort.Strings(allPaths) + for _, path := range allPaths { + fsPath := strings.TrimPrefix(path, "./") + fmt.Fprintf(&b, " • /%s\n", fsPath) + } + + b.WriteString("\n⚠ WARNING:\n") + b.WriteString(" • Existing files at these locations will be OVERWRITTEN\n") + b.WriteString(" • A safety backup will be created before restoration\n") + b.WriteString(" • Services may need to be restarted after restoration\n\n") + if (hasCategoryID(config.SelectedCategories, "pve_access_control") || hasCategoryID(config.SelectedCategories, "pbs_access_control")) && + (!hasCategoryID(config.SelectedCategories, "network") || !hasCategoryID(config.SelectedCategories, "ssl")) { + b.WriteString(" • TFA/WebAuthn: keep the same UI origin (FQDN/hostname and port) for 1:1\n") + b.WriteString(" compatibility, and restore 'network' + 'ssl'\n\n") + } + + return b.String() +} diff --git a/internal/orchestrator/restore_plan_render_test.go b/internal/orchestrator/restore_plan_render_test.go new file mode 100644 index 00000000..e9e8d6f7 --- /dev/null +++ b/internal/orchestrator/restore_plan_render_test.go @@ -0,0 +1,68 @@ +package orchestrator + +import ( + "strings" + "testing" + + "charm.land/lipgloss/v2" +) + +// The restore plan is shown in a non-wrapping Pager, so an over-width line is +// truncated (only reachable via horizontal scroll). Every static plan line must +// fit a conventional 80-column terminal. The TFA/WebAuthn advisory is the tightest +// line; a config that arms it plus short paths must keep every line within 80. +func TestBuildRestorePlanTextLinesFit80Columns(t *testing.T) { + config := &SelectiveRestoreConfig{ + Mode: RestoreModeCustom, + SystemType: SystemTypePVE, + SelectedCategories: []Category{ + // pve_access_control without network+ssl arms the TFA/WebAuthn advisory. + {ID: "pve_access_control", Name: "Access control", Description: "Users, roles, TFA", Paths: []string{"./etc/pve/user.cfg"}}, + }, + } + + text := buildRestorePlanText(config) + + if !strings.Contains(text, "TFA/WebAuthn") { + t.Fatalf("expected the TFA/WebAuthn advisory to be present:\n%s", text) + } + for _, line := range strings.Split(text, "\n") { + if w := lipgloss.Width(line); w > 80 { + t.Errorf("plan line exceeds 80 columns (%d): %q", w, line) + } + } +} + +func TestBuildRestorePlanText(t *testing.T) { + config := &SelectiveRestoreConfig{ + Mode: RestoreModeCustom, + SystemType: SystemTypePVE, + SelectedCategories: []Category{ + {Name: "Alpha", Description: "First", Paths: []string{"./etc/alpha"}}, + {Name: "Beta", Description: "Second", Paths: []string{"./var/beta"}}, + }, + } + + text := buildRestorePlanText(config) + + if !strings.Contains(text, "CUSTOM selection (2 categories)") { + t.Fatalf("missing mode line: %s", text) + } + if !strings.Contains(text, "System type: Proxmox Virtual Environment (PVE)") { + t.Fatalf("missing system type line: %s", text) + } + if !strings.Contains(text, "1. Alpha") || !strings.Contains(text, "2. Beta") { + t.Fatalf("missing category entries: %s", text) + } + alphaIndex := strings.Index(text, "/etc/alpha") + betaIndex := strings.Index(text, "/var/beta") + if alphaIndex == -1 || betaIndex == -1 { + t.Fatalf("missing paths: %s", text) + } + if alphaIndex > betaIndex { + t.Fatalf("paths not sorted: %d vs %d", alphaIndex, betaIndex) + } + if !strings.Contains(text, "Existing files at these locations will be OVERWRITTEN") { + t.Fatalf("missing warning text") + } +} diff --git a/internal/orchestrator/restore_tui.go b/internal/orchestrator/restore_tui.go index 331f39e0..ae5f0332 100644 --- a/internal/orchestrator/restore_tui.go +++ b/internal/orchestrator/restore_tui.go @@ -102,62 +102,3 @@ func filterAndSortCategoriesForSystem(available []Category, systemType SystemTyp return relevant } - -func buildRestorePlanText(config *SelectiveRestoreConfig) string { - if config == nil { - return "" - } - - var b strings.Builder - - // No ASCII banner: the Pager renders the styled "Restore plan" title, and the - // legacy box rule was cosmetically inconsistent with the Charm screens. - modeName := "" - switch config.Mode { - case RestoreModeFull: - modeName = "FULL restore (all categories)" - case RestoreModeStorage: - if config.SystemType.SupportsPVE() && !config.SystemType.SupportsPBS() { - modeName = "STORAGE only (cluster + storage + jobs + mounts)" - } else if config.SystemType.SupportsPBS() && !config.SystemType.SupportsPVE() { - modeName = "DATASTORE only (datastores + jobs + mounts)" - } else { - modeName = "STORAGE/DATASTORE only (PVE + PBS storage/jobs + mounts)" - } - case RestoreModeBase: - modeName = "SYSTEM BASE only (network + SSL + SSH + services + filesystem)" - case RestoreModeCustom: - modeName = fmt.Sprintf("CUSTOM selection (%d categories)", len(config.SelectedCategories)) - default: - modeName = "Unknown mode" - } - - fmt.Fprintf(&b, "Restore mode: %s\n", modeName) - fmt.Fprintf(&b, "System type: %s\n\n", GetSystemTypeString(config.SystemType)) - - b.WriteString("Categories to restore:\n") - for i, cat := range config.SelectedCategories { - fmt.Fprintf(&b, " %d. %s\n", i+1, cat.Name) - fmt.Fprintf(&b, " %s\n", cat.Description) - } - - b.WriteString("\nFiles/directories that will be restored:\n") - allPaths := GetSelectedPaths(config.SelectedCategories) - sort.Strings(allPaths) - for _, path := range allPaths { - fsPath := strings.TrimPrefix(path, "./") - fmt.Fprintf(&b, " • /%s\n", fsPath) - } - - b.WriteString("\n⚠ WARNING:\n") - b.WriteString(" • Existing files at these locations will be OVERWRITTEN\n") - b.WriteString(" • A safety backup will be created before restoration\n") - b.WriteString(" • Services may need to be restarted after restoration\n\n") - if (hasCategoryID(config.SelectedCategories, "pve_access_control") || hasCategoryID(config.SelectedCategories, "pbs_access_control")) && - (!hasCategoryID(config.SelectedCategories, "network") || !hasCategoryID(config.SelectedCategories, "ssl")) { - b.WriteString(" • TFA/WebAuthn: keep the same UI origin (FQDN/hostname and port) for 1:1\n") - b.WriteString(" compatibility, and restore 'network' + 'ssl'\n\n") - } - - return b.String() -} diff --git a/internal/orchestrator/restore_tui_test.go b/internal/orchestrator/restore_tui_test.go index 21b939f2..816aa7f3 100644 --- a/internal/orchestrator/restore_tui_test.go +++ b/internal/orchestrator/restore_tui_test.go @@ -1,10 +1,7 @@ package orchestrator import ( - "strings" "testing" - - "charm.land/lipgloss/v2" ) func TestFilterAndSortCategoriesForSystem(t *testing.T) { @@ -36,63 +33,3 @@ func TestFilterAndSortCategoriesForSystem(t *testing.T) { }) } } - -// The restore plan is shown in a non-wrapping Pager, so an over-width line is -// truncated (only reachable via horizontal scroll). Every static plan line must -// fit a conventional 80-column terminal. The TFA/WebAuthn advisory is the tightest -// line; a config that arms it plus short paths must keep every line within 80. -func TestBuildRestorePlanTextLinesFit80Columns(t *testing.T) { - config := &SelectiveRestoreConfig{ - Mode: RestoreModeCustom, - SystemType: SystemTypePVE, - SelectedCategories: []Category{ - // pve_access_control without network+ssl arms the TFA/WebAuthn advisory. - {ID: "pve_access_control", Name: "Access control", Description: "Users, roles, TFA", Paths: []string{"./etc/pve/user.cfg"}}, - }, - } - - text := buildRestorePlanText(config) - - if !strings.Contains(text, "TFA/WebAuthn") { - t.Fatalf("expected the TFA/WebAuthn advisory to be present:\n%s", text) - } - for _, line := range strings.Split(text, "\n") { - if w := lipgloss.Width(line); w > 80 { - t.Errorf("plan line exceeds 80 columns (%d): %q", w, line) - } - } -} - -func TestBuildRestorePlanText(t *testing.T) { - config := &SelectiveRestoreConfig{ - Mode: RestoreModeCustom, - SystemType: SystemTypePVE, - SelectedCategories: []Category{ - {Name: "Alpha", Description: "First", Paths: []string{"./etc/alpha"}}, - {Name: "Beta", Description: "Second", Paths: []string{"./var/beta"}}, - }, - } - - text := buildRestorePlanText(config) - - if !strings.Contains(text, "CUSTOM selection (2 categories)") { - t.Fatalf("missing mode line: %s", text) - } - if !strings.Contains(text, "System type: Proxmox Virtual Environment (PVE)") { - t.Fatalf("missing system type line: %s", text) - } - if !strings.Contains(text, "1. Alpha") || !strings.Contains(text, "2. Beta") { - t.Fatalf("missing category entries: %s", text) - } - alphaIndex := strings.Index(text, "/etc/alpha") - betaIndex := strings.Index(text, "/var/beta") - if alphaIndex == -1 || betaIndex == -1 { - t.Fatalf("missing paths: %s", text) - } - if alphaIndex > betaIndex { - t.Fatalf("paths not sorted: %d vs %d", alphaIndex, betaIndex) - } - if !strings.Contains(text, "Existing files at these locations will be OVERWRITTEN") { - t.Fatalf("missing warning text") - } -} diff --git a/internal/orchestrator/selective.go b/internal/orchestrator/selective.go index 2c47fb6d..77dbaf35 100644 --- a/internal/orchestrator/selective.go +++ b/internal/orchestrator/selective.go @@ -352,69 +352,6 @@ func filterOutExportOnly(categories []Category) []Category { return out } -// ShowRestorePlan displays a detailed plan of what will be restored -func ShowRestorePlan(logger *logging.Logger, config *SelectiveRestoreConfig) { - fmt.Println() - fmt.Println("═══════════════════════════════════════════════════════════════") - fmt.Println("RESTORE PLAN") - fmt.Println("═══════════════════════════════════════════════════════════════") - fmt.Println() - - // Show mode - modeName := "" - switch config.Mode { - case RestoreModeFull: - modeName = "FULL restore (all categories)" - case RestoreModeStorage: - if config.SystemType.SupportsPVE() && !config.SystemType.SupportsPBS() { - modeName = "STORAGE only (cluster + storage + jobs + mounts)" - } else if config.SystemType.SupportsPBS() && !config.SystemType.SupportsPVE() { - modeName = "DATASTORE only (datastores + jobs + mounts)" - } else { - modeName = "STORAGE/DATASTORE only (PVE + PBS storage/jobs + mounts)" - } - case RestoreModeBase: - modeName = "SYSTEM BASE only (network + SSL + SSH + services + filesystem)" - case RestoreModeCustom: - modeName = fmt.Sprintf("CUSTOM selection (%d categories)", len(config.SelectedCategories)) - } - - fmt.Printf("Restore mode: %s\n", modeName) - fmt.Printf("System type: %s\n", GetSystemTypeString(config.SystemType)) - fmt.Println() - - // Show selected categories - fmt.Println("Categories to restore:") - for i, cat := range config.SelectedCategories { - fmt.Printf(" %d. %s\n", i+1, cat.Name) - fmt.Printf(" %s\n", cat.Description) - } - - fmt.Println() - fmt.Println("Files/directories that will be restored:") - - // Collect and display all paths - allPaths := GetSelectedPaths(config.SelectedCategories) - sort.Strings(allPaths) - - for _, path := range allPaths { - // Convert to filesystem path for display - fsPath := strings.TrimPrefix(path, "./") - fmt.Printf(" • /%s\n", fsPath) - } - - fmt.Println() - fmt.Println("⚠ WARNING:") - fmt.Println(" • Existing files at these locations will be OVERWRITTEN") - fmt.Println(" • A safety backup will be created before restoration") - fmt.Println(" • Services may need to be restarted after restoration") - if (hasCategoryID(config.SelectedCategories, "pve_access_control") || hasCategoryID(config.SelectedCategories, "pbs_access_control")) && - (!hasCategoryID(config.SelectedCategories, "network") || !hasCategoryID(config.SelectedCategories, "ssl")) { - fmt.Println(" • TFA/WebAuthn: for best 1:1 compatibility keep the same UI origin (FQDN/hostname and port) and restore 'network' + 'ssl'") - } - fmt.Println() -} - // ConfirmRestoreOperation asks for user confirmation before proceeding func ConfirmRestoreOperation(ctx context.Context, logger *logging.Logger) (bool, error) { return ConfirmRestoreOperationWithReader(ctx, bufio.NewReader(os.Stdin), logger) diff --git a/internal/orchestrator/workflow_ui_cli.go b/internal/orchestrator/workflow_ui_cli.go index 0135c652..59b07a02 100644 --- a/internal/orchestrator/workflow_ui_cli.go +++ b/internal/orchestrator/workflow_ui_cli.go @@ -249,8 +249,30 @@ func (u *cliWorkflowUI) SelectPBSRestoreBehavior(ctx context.Context) (PBSRestor } } +// ShowRestorePlan prints the CLI chrome (ASCII banner) around the plan body +// shared with the Charm front-end (buildRestorePlanText). +// +// Output deliberately stays on os.Stdout rather than u.w(). The CLI restore +// prompts are already split across both sinks -- ConfirmRestoreOperationWithReader +// (selective.go) writes to stdout, while ConfirmRestore's own overwrite prompt +// below writes to u.w() -- so moving only the plan would not unify anything. +// Routing the whole restore prompt set onto u.w() is deferred to its own change; +// note that TestShowRestorePlanOutputsPaths captures os.Stdout and would have to +// move with it. func (u *cliWorkflowUI) ShowRestorePlan(ctx context.Context, config *SelectiveRestoreConfig) error { - ShowRestorePlan(u.logger, config) + if config == nil { + return fmt.Errorf("restore configuration not available") + } + + fmt.Println() + fmt.Println("═══════════════════════════════════════════════════════════════") + fmt.Println("RESTORE PLAN") + fmt.Println("═══════════════════════════════════════════════════════════════") + fmt.Println() + + // Print, not Printf: category paths reach the body verbatim and a '%' in a + // path would otherwise be consumed as a format verb. + fmt.Print(buildRestorePlanText(config)) return nil } diff --git a/internal/orchestrator/workflow_ui_cli_test.go b/internal/orchestrator/workflow_ui_cli_test.go index 0759e3ad..a2bb1b40 100644 --- a/internal/orchestrator/workflow_ui_cli_test.go +++ b/internal/orchestrator/workflow_ui_cli_test.go @@ -258,3 +258,14 @@ func TestPromptOptionAgeAbortsWhenIdle(t *testing.T) { t.Fatalf("idle age prompt must map to a graceful abort (ErrAgeRecipientSetupAborted); got %v", err) } } + +// A nil plan config must be reported as an error, not dereferenced. The free +// ShowRestorePlan this method replaced had no guard and panicked; the Charm +// sibling's guard is pinned by workflow_ui_charm_restore_test.go, so pin the CLI +// one too rather than leaving the two front-ends asymmetrically covered. +func TestCLIWorkflowUIShowRestorePlanRejectsNilConfig(t *testing.T) { + ui := newCLIWorkflowUI(nil, logging.New(types.LogLevelNone, false)) + if err := ui.ShowRestorePlan(context.Background(), nil); err == nil { + t.Fatal("nil config must error") + } +} From 8b7cab1e6be087c89543beca9335e6099f7759ee Mon Sep 17 00:00:00 2001 From: tis24dev Date: Sun, 2 Aug 2026 22:08:31 +0200 Subject: [PATCH 02/50] refactor(install): collapse four copy-paste duplications onto shared code Four helpers existed twice, in each case alongside a shared implementation that already had the wider caller base: - unsetEnvValue in cmd/proxsave was byte-identical to the installer's UnsetEnvValueInTemplate; the cmd copy had exactly one caller. - The healthcheck ping-URL validator existed as validateHealthcheckPingURLCLI in cmd/proxsave and validateHealthcheckPingURL in internal/ui/flows/install. Neither location could serve both -- cmd must not import internal/ui/flows -- so the pair is promoted to internal/installer next to DeriveHealthcheckSelfParams, which both consumers already import. - ShowCategorySelectionMenuWithReader inlined the category filter+sort that filterAndSortCategoriesForSystem already provides in the same package; the two differed by a make() capacity hint and nothing else. CLI and Charm restore now share one filter, so its existing test guards both front-ends. - runPostInstallAuditCLI hand-rolled read -> setEnvValue(KEY,false) -> atomic write, duplicating installer.ApplyAuditDisables down to the .tmp.audit suffix. WriteConfigFileAtomic's temp-cleanup defer moves ahead of root.WriteFile. It was registered after the write, so a failing or partial write (ENOSPC, EIO, EDQUOT) returned with the defer never registered and orphaned the temp file. The callers in cmd/ papered over this with their own outer defers; moving it here fixes the window for all callers and lets those outer defers go. A new test pins it -- verified to fail with the defer in its old position. Note one behavior change: the audit's failure text loses its read-vs-write distinction, since ApplyAuditDisables returns an already-wrapped error. The control flow is unchanged -- both arms still warn and return nil, non-blocking. sort.Strings(keys) is deliberately kept in runPostInstallAuditCLI: it orders the summary line printed just below, and it is that file's only use of the sort package. --- cmd/proxsave/config_helpers.go | 31 ---------- cmd/proxsave/install.go | 57 +++---------------- internal/installer/install_data.go | 38 +++++++++++++ internal/installer/ui_support.go | 6 +- internal/installer/ui_support_test.go | 27 +++++++++ internal/orchestrator/selective.go | 25 +------- .../flows/install/healthcheck_selfparams.go | 52 +++-------------- .../install/healthcheck_selfparams_test.go | 12 ++-- 8 files changed, 92 insertions(+), 156 deletions(-) diff --git a/cmd/proxsave/config_helpers.go b/cmd/proxsave/config_helpers.go index 28b91b30..0eb7d92c 100644 --- a/cmd/proxsave/config_helpers.go +++ b/cmd/proxsave/config_helpers.go @@ -49,37 +49,6 @@ func setEnvValue(template, key, value string) string { return utils.SetEnvValue(template, key, value) } -func unsetEnvValue(template, key string) string { - key = strings.TrimSpace(key) - if key == "" { - return template - } - - lines := strings.Split(template, "\n") - out := make([]string, 0, len(lines)) - for _, line := range lines { - trimmed := strings.TrimSpace(line) - if utils.IsComment(trimmed) { - out = append(out, line) - continue - } - parts := strings.SplitN(trimmed, "=", 2) - if len(parts) != 2 { - out = append(out, line) - continue - } - parsedKey := strings.TrimSpace(parts[0]) - if fields := strings.Fields(parsedKey); len(fields) >= 2 && fields[0] == "export" { - parsedKey = fields[1] - } - if strings.EqualFold(parsedKey, key) { - continue - } - out = append(out, line) - } - return strings.Join(out, "\n") -} - func sanitizeEnvValue(value string) string { value = strings.Map(func(r rune) rune { if r == '\n' || r == '\r' || r == '\x00' { diff --git a/cmd/proxsave/install.go b/cmd/proxsave/install.go index 1e45c23d..04678323 100644 --- a/cmd/proxsave/install.go +++ b/cmd/proxsave/install.go @@ -5,7 +5,6 @@ import ( "context" "errors" "fmt" - neturl "net/url" "os" "os/exec" "path/filepath" @@ -306,27 +305,14 @@ func runPostInstallAuditCLI(ctx context.Context, reader *bufio.Reader, execPath, return nil } - contentBytes, err := safefs.ReadFileUnderRoot(configPath) - if err != nil { - fmt.Printf("ERROR: Unable to update configuration (read failed): %v\n", err) - if bootstrap != nil { - bootstrap.Warning("Post-install audit: unable to update configuration (read failed): %v", err) - } - return nil - } - content := string(contentBytes) - sort.Strings(keys) - for _, key := range keys { - content = setEnvValue(content, key, "false") - } - - tmpAuditPath := configPath + ".tmp.audit" - defer cleanupTempConfig(tmpAuditPath) - if err := writeConfigFile(configPath, tmpAuditPath, content); err != nil { - fmt.Printf("ERROR: Unable to update configuration (write failed): %v\n", err) + // The summary printed just below renders the un-normalized `keys` slice while the + // file receives the ToUpper-normalized keys ApplyAuditDisables writes; the two agree + // only because internal/installer/audit.go already uppercases each Key at the source. + if err := installer.ApplyAuditDisables(configPath, keys); err != nil { + fmt.Printf("ERROR: Unable to update configuration: %v\n", err) if bootstrap != nil { - bootstrap.Warning("Post-install audit: unable to update configuration (write failed): %v", err) + bootstrap.Warning("Post-install audit: unable to update configuration: %v", err) } return nil } @@ -901,7 +887,7 @@ func configureNotifications(ctx context.Context, reader *bufio.Reader, template } template = setEnvValue(template, "EMAIL_ENABLED", "true") template = setEnvValue(template, "EMAIL_DELIVERY_METHOD", method) - template = unsetEnvValue(template, "EMAIL_FALLBACK_PMF") + template = installer.UnsetEnvValueInTemplate(template, "EMAIL_FALLBACK_PMF") template = setEnvValue(template, "EMAIL_FALLBACK_SENDMAIL", "true") } else { template = setEnvValue(template, "EMAIL_ENABLED", "false") @@ -1050,31 +1036,6 @@ func configureHealthcheckMode(ctx context.Context, reader *bufio.Reader, def str } } -// validateHealthcheckPingURLCLI is the CLI-side ping-URL validator, identical in -// intent to the TUI's validateHealthcheckPingURL: an absolute http(s) URL with a -// host. It is used for the required alive/backup URLs (empty rejected) via -// promptNonEmpty's retry loop and, wrapped, for the optional URLs. -func validateHealthcheckPingURLCLI(v string) error { - v = strings.TrimSpace(v) - if v == "" { - return fmt.Errorf("cannot be empty") - } - if !strings.HasPrefix(v, "http://") && !strings.HasPrefix(v, "https://") { - return fmt.Errorf("URL must start with http:// or https://") - } - u, err := neturl.ParseRequestURI(v) - if err != nil { - return fmt.Errorf("invalid URL: %v", err) - } - if u.Scheme != "http" && u.Scheme != "https" { - return fmt.Errorf("URL must start with http:// or https://") - } - if u.Host == "" { - return fmt.Errorf("URL must include a host") - } - return nil -} - // promptHealthcheckRequiredURL prompts for a required ping URL, re-asking until the // value is a valid http(s) URL (parity with the TUI required-field validator). func promptHealthcheckRequiredURL(ctx context.Context, reader *bufio.Reader, question, def string) (string, error) { @@ -1084,7 +1045,7 @@ func promptHealthcheckRequiredURL(ctx context.Context, reader *bufio.Reader, que return "", err } val = sanitizeEnvValue(val) - if verr := validateHealthcheckPingURLCLI(val); verr != nil { + if verr := installer.ValidateHealthcheckPingURL(val); verr != nil { fmt.Printf("%v\n", verr) continue } @@ -1104,7 +1065,7 @@ func promptHealthcheckOptionalURL(ctx context.Context, reader *bufio.Reader, que if strings.TrimSpace(val) == "" { return "", nil } - if verr := validateHealthcheckPingURLCLI(val); verr != nil { + if verr := installer.ValidateHealthcheckPingURL(val); verr != nil { fmt.Printf("%v\n", verr) continue } diff --git a/internal/installer/install_data.go b/internal/installer/install_data.go index 0b75ef95..eb6639a3 100644 --- a/internal/installer/install_data.go +++ b/internal/installer/install_data.go @@ -10,6 +10,7 @@ import ( "bufio" "errors" "fmt" + "net/url" "strings" "github.com/tis24dev/proxsave/internal/config" @@ -137,6 +138,43 @@ func DeriveHealthcheckSelfParams(template string) HealthcheckSelfParams { } } +// ValidateHealthcheckPingURL accepts only a well-formed absolute http(s) ping URL +// with a host. It mirrors the http(s) gate style of serverbot.SanitizeLoginURL but +// is a full-URL validator: an empty value is rejected (use it on required fields; +// wrap it for optional ones). Callers paste the ENTIRE ping URL of each check +// (e.g. https://hc-ping.com/), so the daemon's selfURLs() full-URL branch +// resolves it verbatim. Shared by the CLI wizard and the Charm install flow. +func ValidateHealthcheckPingURL(v string) error { + v = strings.TrimSpace(v) + if v == "" { + return fmt.Errorf("cannot be empty") + } + if !strings.HasPrefix(v, "http://") && !strings.HasPrefix(v, "https://") { + return fmt.Errorf("URL must start with http:// or https://") + } + u, err := url.ParseRequestURI(v) + if err != nil { + return fmt.Errorf("invalid URL: %v", err) + } + if u.Scheme != "http" && u.Scheme != "https" { + return fmt.Errorf("URL must start with http:// or https://") + } + if u.Host == "" { + return fmt.Errorf("URL must include a host") + } + return nil +} + +// ValidateOptionalHealthcheckPingURL is ValidateHealthcheckPingURL for optional +// fields: an empty value is accepted (the sensor is simply not configured), a +// non-empty value must still be a valid http(s) URL. +func ValidateOptionalHealthcheckPingURL(v string) error { + if strings.TrimSpace(v) == "" { + return nil + } + return ValidateHealthcheckPingURL(v) +} + // ExistingConfigAction represents how to handle an already-present configuration file. // If baseTemplate is empty, the embedded default template is used. diff --git a/internal/installer/ui_support.go b/internal/installer/ui_support.go index b613b9ea..f1b3d86d 100644 --- a/internal/installer/ui_support.go +++ b/internal/installer/ui_support.go @@ -124,14 +124,14 @@ func WriteConfigFileAtomic(configPath, tmpPath, content string) error { return fmt.Errorf("failed to open configuration directory: %w", err) } defer func() { _ = root.Close() }() - if err := root.WriteFile(filepath.Base(tmpPath), []byte(content), 0o600); err != nil { - return fmt.Errorf("failed to write configuration file: %w", err) - } defer func() { if _, statErr := os.Stat(tmpPath); statErr == nil { _ = os.Remove(tmpPath) } }() + if err := root.WriteFile(filepath.Base(tmpPath), []byte(content), 0o600); err != nil { + return fmt.Errorf("failed to write configuration file: %w", err) + } if err := os.Rename(tmpPath, configPath); err != nil { return fmt.Errorf("failed to finalize configuration file: %w", err) } diff --git a/internal/installer/ui_support_test.go b/internal/installer/ui_support_test.go index 3f4f5e9d..24a30393 100644 --- a/internal/installer/ui_support_test.go +++ b/internal/installer/ui_support_test.go @@ -55,3 +55,30 @@ func TestWriteConfigFileAtomicWritesAbsolutePath(t *testing.T) { t.Fatalf("temp file should be gone after rename, stat err=%v", err) } } + +// A FAILED write must not leave the temp entry behind either. The cleanup defer +// is registered BEFORE root.WriteFile precisely so this path is covered: with the +// defer registered after the write (as it was before the callers in cmd/ dropped +// their own outer cleanup defers), an ENOSPC/EIO/EDQUOT write returned with the +// defer never registered and orphaned the temp file. tmpPath is pre-created as an +// empty directory here, which is the portable way to make root.WriteFile fail. +func TestWriteConfigFileAtomicCleansUpAfterFailedWrite(t *testing.T) { + dir := t.TempDir() + configPath := filepath.Join(dir, "backup.env") + tmpPath := configPath + ".tmp" + + if err := os.Mkdir(tmpPath, 0o700); err != nil { + t.Fatalf("seed temp path as a directory: %v", err) + } + + if err := WriteConfigFileAtomic(configPath, tmpPath, "KEY=value\n"); err == nil { + t.Fatal("writing over a directory must fail") + } + + if _, err := os.Stat(tmpPath); !os.IsNotExist(err) { + t.Fatalf("temp entry must be removed after a failed write, stat err=%v", err) + } + if _, err := os.Stat(configPath); !os.IsNotExist(err) { + t.Fatalf("config must not exist after a failed write, stat err=%v", err) + } +} diff --git a/internal/orchestrator/selective.go b/internal/orchestrator/selective.go index 77dbaf35..0e6b7527 100644 --- a/internal/orchestrator/selective.go +++ b/internal/orchestrator/selective.go @@ -8,7 +8,6 @@ import ( "fmt" "os" "path" - "sort" "strconv" "strings" @@ -179,28 +178,8 @@ func ShowCategorySelectionMenuWithReader(ctx context.Context, reader *bufio.Read reader = bufio.NewReader(os.Stdin) } - // Filter categories by system type - relevantCategories := make([]Category, 0) - for _, cat := range availableCategories { - if cat.Type == CategoryTypeCommon || - (systemType.SupportsPVE() && cat.Type == CategoryTypePVE) || - (systemType.SupportsPBS() && cat.Type == CategoryTypePBS) { - relevantCategories = append(relevantCategories, cat) - } - } - - // Sort categories: PVE/PBS first, then common - sort.Slice(relevantCategories, func(i, j int) bool { - if relevantCategories[i].Type != relevantCategories[j].Type { - if relevantCategories[i].Type == CategoryTypeCommon { - return false - } - if relevantCategories[j].Type == CategoryTypeCommon { - return true - } - } - return relevantCategories[i].Name < relevantCategories[j].Name - }) + // Filter by system type and sort (PVE/PBS first, then common). + relevantCategories := filterAndSortCategoriesForSystem(availableCategories, systemType) // Track selection state selected := make(map[int]bool) diff --git a/internal/ui/flows/install/healthcheck_selfparams.go b/internal/ui/flows/install/healthcheck_selfparams.go index 00a6f497..a92f0db6 100644 --- a/internal/ui/flows/install/healthcheck_selfparams.go +++ b/internal/ui/flows/install/healthcheck_selfparams.go @@ -3,7 +3,6 @@ package install import ( "context" "fmt" - "net/url" "os" "strings" @@ -12,43 +11,6 @@ import ( "github.com/tis24dev/proxsave/internal/ui/shell" ) -// validateHealthcheckPingURL accepts only a well-formed absolute http(s) ping URL -// with a host. It mirrors the http(s) gate style of serverbot.SanitizeLoginURL but -// is a full-URL validator: an empty value is rejected (use it on required fields; -// wrap it for optional ones). Callers paste the ENTIRE ping URL of each check -// (e.g. https://hc-ping.com/), so the daemon's selfURLs() full-URL branch -// resolves it verbatim. -func validateHealthcheckPingURL(v string) error { - v = strings.TrimSpace(v) - if v == "" { - return fmt.Errorf("cannot be empty") - } - if !strings.HasPrefix(v, "http://") && !strings.HasPrefix(v, "https://") { - return fmt.Errorf("URL must start with http:// or https://") - } - u, err := url.ParseRequestURI(v) - if err != nil { - return fmt.Errorf("invalid URL: %v", err) - } - if u.Scheme != "http" && u.Scheme != "https" { - return fmt.Errorf("URL must start with http:// or https://") - } - if u.Host == "" { - return fmt.Errorf("URL must include a host") - } - return nil -} - -// validateOptionalHealthcheckPingURL is validateHealthcheckPingURL for optional -// fields: an empty value is accepted (the sensor is simply not configured), a -// non-empty value must still be a valid http(s) URL. -func validateOptionalHealthcheckPingURL(v string) error { - if strings.TrimSpace(v) == "" { - return nil - } - return validateHealthcheckPingURL(v) -} - // RunHealthcheckSelfParams shows the self-mode healthchecks parameters screen: one // aligned form collecting the FULL ping URLs of every sensor. Alive + Backup are // REQUIRED; updates and the four notify URLs are OPTIONAL. It prefills from the @@ -71,49 +33,49 @@ func RunHealthcheckSelfParams(ctx context.Context, session *shell.Session, baseD Description: "HEALTHCHECK_ALIVE_URL (required): the full service-alive ping URL (e.g. https://hc-ping.com/).", Kind: components.FieldText, Text: prefill.AliveURL, - Validate: validateHealthcheckPingURL, + Validate: installer.ValidateHealthcheckPingURL, } backup := &components.FormField{ Label: "Backup ping URL", Description: "HEALTHCHECK_BACKUP_URL (required): the full backup-outcome ping URL.", Kind: components.FieldText, Text: prefill.BackupURL, - Validate: validateHealthcheckPingURL, + Validate: installer.ValidateHealthcheckPingURL, } updates := &components.FormField{ Label: "Updates ping URL", Description: "HEALTHCHECK_UPDATES_URL (optional): the updates-check ping URL.", Kind: components.FieldText, Text: prefill.UpdatesURL, - Validate: validateOptionalHealthcheckPingURL, + Validate: installer.ValidateOptionalHealthcheckPingURL, } notifyEmail := &components.FormField{ Label: "Notify email URL", Description: "HEALTHCHECK_NOTIFY_EMAIL_URL (optional): the email-notification ping URL.", Kind: components.FieldText, Text: prefill.NotifyEmailURL, - Validate: validateOptionalHealthcheckPingURL, + Validate: installer.ValidateOptionalHealthcheckPingURL, } notifyTelegram := &components.FormField{ Label: "Notify Telegram URL", Description: "HEALTHCHECK_NOTIFY_TELEGRAM_URL (optional): the Telegram-notification ping URL.", Kind: components.FieldText, Text: prefill.NotifyTelegramURL, - Validate: validateOptionalHealthcheckPingURL, + Validate: installer.ValidateOptionalHealthcheckPingURL, } notifyGotify := &components.FormField{ Label: "Notify Gotify URL", Description: "HEALTHCHECK_NOTIFY_GOTIFY_URL (optional): the Gotify-notification ping URL.", Kind: components.FieldText, Text: prefill.NotifyGotifyURL, - Validate: validateOptionalHealthcheckPingURL, + Validate: installer.ValidateOptionalHealthcheckPingURL, } notifyWebhook := &components.FormField{ Label: "Notify webhook URL", Description: "HEALTHCHECK_NOTIFY_WEBHOOK_URL (optional): the webhook-notification ping URL.", Kind: components.FieldText, Text: prefill.NotifyWebhookURL, - Validate: validateOptionalHealthcheckPingURL, + Validate: installer.ValidateOptionalHealthcheckPingURL, } fields := []*components.FormField{ diff --git a/internal/ui/flows/install/healthcheck_selfparams_test.go b/internal/ui/flows/install/healthcheck_selfparams_test.go index 0ab5caae..e475713b 100644 --- a/internal/ui/flows/install/healthcheck_selfparams_test.go +++ b/internal/ui/flows/install/healthcheck_selfparams_test.go @@ -29,10 +29,10 @@ func TestValidateHealthcheckPingURL(t *testing.T) { "https://hc.example.org/a-slug", } for _, v := range valid { - if err := validateHealthcheckPingURL(v); err != nil { + if err := installer.ValidateHealthcheckPingURL(v); err != nil { t.Errorf("required: %q must be valid, got %v", v, err) } - if err := validateOptionalHealthcheckPingURL(v); err != nil { + if err := installer.ValidateOptionalHealthcheckPingURL(v); err != nil { t.Errorf("optional: %q must be valid, got %v", v, err) } } @@ -45,19 +45,19 @@ func TestValidateHealthcheckPingURL(t *testing.T) { "not a url", // junk } for _, v := range bad { - if err := validateHealthcheckPingURL(v); err == nil { + if err := installer.ValidateHealthcheckPingURL(v); err == nil { t.Errorf("required: %q must be rejected", v) } } // Optional accepts empty but still rejects a malformed non-empty value. - if err := validateOptionalHealthcheckPingURL(""); err != nil { + if err := installer.ValidateOptionalHealthcheckPingURL(""); err != nil { t.Errorf("optional empty must be accepted, got %v", err) } - if err := validateOptionalHealthcheckPingURL(" "); err != nil { + if err := installer.ValidateOptionalHealthcheckPingURL(" "); err != nil { t.Errorf("optional blank must be accepted, got %v", err) } - if err := validateOptionalHealthcheckPingURL("ftp://x/y"); err == nil { + if err := installer.ValidateOptionalHealthcheckPingURL("ftp://x/y"); err == nil { t.Error("optional must still reject a malformed non-empty URL") } } From b444b320f23eae8176e756d1b257f010a9f5244d Mon Sep 17 00:00:00 2001 From: tis24dev Date: Sun, 2 Aug 2026 22:09:57 +0200 Subject: [PATCH 03/50] refactor(install): drop cmd-local writeConfigFile for installer.WriteConfigFileAtomic The two writers took the same arguments and did the same thing, except the shared one is stricter: it rejects a bare-filename config path, which would otherwise write the config into the process working directory, and it removes the temp file when the rename fails. Three call sites remained after the audit path stopped writing directly. One of them, setBackupEnvKeys in daemon_setup.go, is the only caller that actually leaked backup.env.daemon.tmp on a failed rename. The stricter path check is not reachable from these call sites: every config path is absolutized by resolveInstallConfigPath before mode dispatch, and fallbackBaseDir always returns an absolute base. The install_tui.go inline temp-cleanup defer goes too -- WriteConfigFileAtomic now covers the write failure window as well, since the previous commit moved its cleanup defer ahead of the write. --- cmd/proxsave/daemon_setup.go | 2 +- cmd/proxsave/install.go | 25 +------------------------ cmd/proxsave/install_tui.go | 8 +------- 3 files changed, 3 insertions(+), 32 deletions(-) diff --git a/cmd/proxsave/daemon_setup.go b/cmd/proxsave/daemon_setup.go index 7ce4fe6c..e6c023f8 100644 --- a/cmd/proxsave/daemon_setup.go +++ b/cmd/proxsave/daemon_setup.go @@ -339,7 +339,7 @@ func setBackupEnvKeys(configPath string, kv map[string]string) error { for _, k := range keys { content = utils.SetEnvValue(content, k, kv[k]) } - return writeConfigFile(configPath, configPath+".daemon.tmp", content) + return installer.WriteConfigFileAtomic(configPath, configPath+".daemon.tmp", content) } // reconcileSchedulerAfterInstall makes the scheduler engine a MUTUALLY EXCLUSIVE diff --git a/cmd/proxsave/install.go b/cmd/proxsave/install.go index 04678323..3ff3029f 100644 --- a/cmd/proxsave/install.go +++ b/cmd/proxsave/install.go @@ -584,7 +584,7 @@ func runConfigWizardCLI(ctx context.Context, reader *bufio.Reader, configPath, t template = setEnvValue(template, "HEALTHCHECK_MODE", "off") clearHCURLs() } - if err := writeConfigFile(configPath, tmpConfigPath, template); err != nil { + if err := installer.WriteConfigFileAtomic(configPath, tmpConfigPath, template); err != nil { return installConfigResult{}, err } @@ -1162,29 +1162,6 @@ func configureCronTime(ctx context.Context, reader *bufio.Reader, defaultCron st } } -func writeConfigFile(configPath, tmpConfigPath, content string) error { - dir := filepath.Dir(configPath) - if err := os.MkdirAll(dir, 0o700); err != nil { - return fmt.Errorf("failed to create configuration directory: %w", err) - } - // Confine the temp write to the configuration directory via os.Root so the - // admin-supplied --config path cannot place the file outside that directory - // (gosec G703 path-traversal containment). tmpConfigPath is configPath with a - // suffix, so it always resolves to a single component within dir. - root, err := os.OpenRoot(dir) - if err != nil { - return fmt.Errorf("failed to open configuration directory: %w", err) - } - defer func() { _ = root.Close() }() - if err := root.WriteFile(filepath.Base(tmpConfigPath), []byte(content), 0o600); err != nil { - return fmt.Errorf("failed to write configuration file: %w", err) - } - if err := os.Rename(tmpConfigPath, configPath); err != nil { - return fmt.Errorf("failed to finalize configuration file: %w", err) - } - return nil -} - func wrapInstallError(err error) error { if err == nil { return nil diff --git a/cmd/proxsave/install_tui.go b/cmd/proxsave/install_tui.go index 42d5929d..8b897799 100644 --- a/cmd/proxsave/install_tui.go +++ b/cmd/proxsave/install_tui.go @@ -185,13 +185,7 @@ func runInstallTUI(ctx context.Context, configPath string, bootstrap *logging.Bo // Write configuration file logging.DebugStepBootstrap(bootstrap, "install workflow (tui)", "writing configuration") tmpConfigPath := configPath + ".tmp" - defer func() { - if _, err := os.Stat(tmpConfigPath); err == nil { - _ = os.Remove(tmpConfigPath) - } - }() - - if err := writeConfigFile(configPath, tmpConfigPath, template); err != nil { + if err := installer.WriteConfigFileAtomic(configPath, tmpConfigPath, template); err != nil { return err } From e73a731c2b18031ae5f9cc967eaec5d86e592f01 Mon Sep 17 00:00:00 2001 From: tis24dev Date: Sun, 2 Aug 2026 22:17:49 +0200 Subject: [PATCH 04/50] refactor(install): one preserved-entries formatter for both front-ends The --new-install confirmation rendered the kept entries twice and the two copies disagreed. The CLI appended exactly one trailing slash per entry; the Charm copy stat'ed filepath.Join(baseDir, entry) and appended the slash only for a directory that already existed, so on a fresh host the destructive confirmation prompt read "build" instead of "build/" -- output varying with host state, inside the prompt that asks to wipe a directory. It also left "build//" unnormalized. The CLI behavior wins and moves to internal/installer as FormatPreservedEntries. The entries are the compile-time set [build env identity], all of them BASE_DIR subdirectories, so the stat asked a question whose answer is already known and answered it wrong when the directory had not been created yet. Removing it also takes the only filesystem access out of a formatting function. ConfirmNewInstall keeps its signature: baseDir is still used for the prompt text, just not for formatting. The CLI table test moves next to the function it now covers. TestFormatPreservedEntriesResolvesAgainstBaseDir is dropped rather than ported -- it guarded resolution against baseDir instead of the working directory, and there is no longer any path resolution to get wrong. --- cmd/proxsave/install.go | 2 +- cmd/proxsave/new_install.go | 20 ++---------- cmd/proxsave/new_install_test.go | 37 --------------------- internal/installer/ui_support.go | 25 ++++++++++++++ internal/installer/ui_support_test.go | 40 +++++++++++++++++++++++ internal/ui/flows/install/install.go | 24 +------------- internal/ui/flows/install/install_test.go | 17 ---------- 7 files changed, 70 insertions(+), 95 deletions(-) diff --git a/cmd/proxsave/install.go b/cmd/proxsave/install.go index 3ff3029f..68cd910b 100644 --- a/cmd/proxsave/install.go +++ b/cmd/proxsave/install.go @@ -355,7 +355,7 @@ func runNewInstall(ctx context.Context, configPath string, bootstrap *logging.Bo } if bootstrap != nil { - bootstrap.Info("Resetting %s (preserving %s)", plan.BaseDir, formatNewInstallPreservedEntries(plan.PreservedEntries)) + bootstrap.Info("Resetting %s (preserving %s)", plan.BaseDir, installer.FormatPreservedEntries(plan.PreservedEntries)) } logging.DebugStepBootstrap(bootstrap, "new-install workflow", "resetting base dir") if err := resetInstallBaseDirWithContext(ctx, plan.BaseDir, bootstrap); err != nil { diff --git a/cmd/proxsave/new_install.go b/cmd/proxsave/new_install.go index 9ee6fb35..3f31883e 100644 --- a/cmd/proxsave/new_install.go +++ b/cmd/proxsave/new_install.go @@ -7,6 +7,8 @@ import ( "os" "sort" "strings" + + "github.com/tis24dev/proxsave/internal/installer" ) type newInstallPlan struct { @@ -53,22 +55,6 @@ func newInstallPreserveSet() map[string]struct{} { return result } -func formatNewInstallPreservedEntries(entries []string) string { - formatted := make([]string, 0, len(entries)) - for _, entry := range entries { - trimmed := strings.TrimSpace(entry) - trimmed = strings.TrimRight(trimmed, "/") - if trimmed == "" { - continue - } - formatted = append(formatted, trimmed+"/") - } - if len(formatted) == 0 { - return "(none)" - } - return strings.Join(formatted, " ") -} - func confirmNewInstallCLI(ctx context.Context, reader *bufio.Reader, plan newInstallPlan) (bool, error) { if reader == nil { reader = bufio.NewReader(os.Stdin) @@ -78,7 +64,7 @@ func confirmNewInstallCLI(ctx context.Context, reader *bufio.Reader, plan newIns fmt.Println("--- New installation reset ---") fmt.Printf("Base directory: %s\n", plan.BaseDir) fmt.Printf("Build signature: %s\n", plan.BuildSignature) - fmt.Printf("Preserved entries: %s\n", formatNewInstallPreservedEntries(plan.PreservedEntries)) + fmt.Printf("Preserved entries: %s\n", installer.FormatPreservedEntries(plan.PreservedEntries)) fmt.Println("Everything else under the base directory will be removed.") return promptYesNo(ctx, reader, "Continue? [y/N]: ", false) diff --git a/cmd/proxsave/new_install_test.go b/cmd/proxsave/new_install_test.go index 8ced7df5..7b4b55d6 100644 --- a/cmd/proxsave/new_install_test.go +++ b/cmd/proxsave/new_install_test.go @@ -101,43 +101,6 @@ func TestBuildNewInstallPlanRejectsEmptyConfigPath(t *testing.T) { } } -func TestFormatNewInstallPreservedEntries(t *testing.T) { - tests := []struct { - name string - entries []string - want string - }{ - { - name: "formats trimmed entries", - entries: []string{" build ", "env", " identity"}, - want: "build/ env/ identity/", - }, - { - name: "returns none for nil input", - entries: nil, - want: "(none)", - }, - { - name: "returns none for blank input", - entries: []string{"", " ", "\t"}, - want: "(none)", - }, - { - name: "normalizes trailing slashes", - entries: []string{"env/", "build//", " identity/// ", "/"}, - want: "env/ build/ identity/", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := formatNewInstallPreservedEntries(tt.entries); got != tt.want { - t.Fatalf("formatNewInstallPreservedEntries(%v) = %q, want %q", tt.entries, got, tt.want) - } - }) - } -} - func TestConfirmNewInstallCLIContinue(t *testing.T) { plan := newInstallPlan{ BaseDir: "/opt/proxsave", diff --git a/internal/installer/ui_support.go b/internal/installer/ui_support.go index f1b3d86d..f2927952 100644 --- a/internal/installer/ui_support.go +++ b/internal/installer/ui_support.go @@ -137,3 +137,28 @@ func WriteConfigFileAtomic(configPath, tmpPath, content string) error { } return nil } + +// FormatPreservedEntries renders the entries a --new-install reset keeps, for +// the confirmation both front-ends show before wiping the base directory. +// +// Every entry gets exactly one trailing slash. The list is the compile-time set +// returned by the caller (build, env, identity), all of them BASE_DIR +// subdirectories, so there is nothing to detect: the Charm copy this replaces +// stat'ed each path and omitted the slash for a directory that did not exist +// yet, which made a destructive confirmation prompt render differently +// depending on host state. If the set ever gains a non-directory entry, the +// unconditional slash becomes wrong for it and this must be revisited. +func FormatPreservedEntries(entries []string) string { + formatted := make([]string, 0, len(entries)) + for _, entry := range entries { + trimmed := strings.TrimRight(strings.TrimSpace(entry), "/") + if trimmed == "" { + continue + } + formatted = append(formatted, trimmed+"/") + } + if len(formatted) == 0 { + return "(none)" + } + return strings.Join(formatted, " ") +} diff --git a/internal/installer/ui_support_test.go b/internal/installer/ui_support_test.go index 24a30393..55a39150 100644 --- a/internal/installer/ui_support_test.go +++ b/internal/installer/ui_support_test.go @@ -82,3 +82,43 @@ func TestWriteConfigFileAtomicCleansUpAfterFailedWrite(t *testing.T) { t.Fatalf("config must not exist after a failed write, stat err=%v", err) } } + +// The preserved set is the compile-time [build env identity], so the renderer +// is pure: one trailing slash per entry, no filesystem lookup. Table moved +// verbatim from the CLI-only copy this function replaced. +func TestFormatPreservedEntries(t *testing.T) { + tests := []struct { + name string + entries []string + want string + }{ + { + name: "formats trimmed entries", + entries: []string{" build ", "env", " identity"}, + want: "build/ env/ identity/", + }, + { + name: "returns none for nil input", + entries: nil, + want: "(none)", + }, + { + name: "returns none for blank input", + entries: []string{"", " ", "\t"}, + want: "(none)", + }, + { + name: "normalizes trailing slashes", + entries: []string{"env/", "build//", " identity/// ", "/"}, + want: "env/ build/ identity/", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := FormatPreservedEntries(tt.entries); got != tt.want { + t.Fatalf("FormatPreservedEntries(%v) = %q, want %q", tt.entries, got, tt.want) + } + }) + } +} diff --git a/internal/ui/flows/install/install.go b/internal/ui/flows/install/install.go index 18718221..0236e932 100644 --- a/internal/ui/flows/install/install.go +++ b/internal/ui/flows/install/install.go @@ -10,7 +10,6 @@ import ( "context" "fmt" "os" - "path/filepath" "strings" "github.com/tis24dev/proxsave/internal/config" @@ -294,7 +293,7 @@ func ConfirmNewInstall(ctx context.Context, session *shell.Session, baseDir stri res, err := shell.Ask(ctx, session, components.NewConfirm( "Confirm new install", fmt.Sprintf("Base directory to reset:\n%s\n\nThis keeps %s\nbut deletes everything else.\n\nContinue?", - baseDir, formatPreservedEntries(baseDir, preservedEntries)), + baseDir, installer.FormatPreservedEntries(preservedEntries)), components.WithLabels("Continue", "Cancel"), components.WithDefaultYes(false), components.WithDanger(), @@ -307,24 +306,3 @@ func ConfirmNewInstall(ctx context.Context, session *shell.Session, baseDir stri } return res.Answer, nil } - -func formatPreservedEntries(baseDir string, entries []string) string { - formatted := make([]string, 0, len(entries)) - for _, entry := range entries { - trimmed := strings.TrimSpace(entry) - if trimmed == "" { - continue - } - if !strings.HasSuffix(trimmed, "/") { - resolved := filepath.Join(baseDir, trimmed) - if fi, err := os.Stat(resolved); err == nil && fi.IsDir() { - trimmed += "/" - } - } - formatted = append(formatted, trimmed) - } - if len(formatted) == 0 { - return "(none)" - } - return strings.Join(formatted, " ") -} diff --git a/internal/ui/flows/install/install_test.go b/internal/ui/flows/install/install_test.go index 0582fec0..b8e9edee 100644 --- a/internal/ui/flows/install/install_test.go +++ b/internal/ui/flows/install/install_test.go @@ -577,23 +577,6 @@ func TestRunPostInstallAuditSkipAndEsc(t *testing.T) { } } -// TestFormatPreservedEntriesResolvesAgainstBaseDir guards the directory -// detection against the CWD (regression salvaged from the deleted wizard -// suite: entries must resolve against baseDir, not the working directory). -func TestFormatPreservedEntriesResolvesAgainstBaseDir(t *testing.T) { - baseDir := t.TempDir() - if err := os.Mkdir(filepath.Join(baseDir, "env"), 0o755); err != nil { - t.Fatal(err) - } - got := formatPreservedEntries(baseDir, []string{"env", "build", " ", ""}) - if got != "env/ build" { - t.Fatalf("formatPreservedEntries = %q, want %q", got, "env/ build") - } - if formatPreservedEntries(baseDir, nil) != "(none)" { - t.Fatal("empty entries must render (none)") - } -} - func TestBuildTelegramPrompt(t *testing.T) { // Linked -> green "✓ LINKED"; Server ID boxed. v := buildTelegramPrompt("123456789", "/id/.server_identity", true, "Linked.", "Linked", orchestrator.TelegramSeveritySuccess, 200) From b8dd4b210c47b874bbb672b33eb930e89da96d45 Mon Sep 17 00:00:00 2001 From: tis24dev Date: Sun, 2 Aug 2026 22:32:36 +0200 Subject: [PATCH 05/50] docs(encryption): fix the claims that cost data in an emergency Four MED audit findings, each re-verified against the source. Two of the four the audit overstated; the corrections follow the code, not the audit. - "no plaintext on disk" was stated in five places and is wrong in all of them. Only the ARCHIVE is never plaintext. Every run stages a full clear copy of the collected files, /etc/shadow and /etc/pve/priv included, under /tmp/proxsave, and the tar is streamed from that directory rather than from memory. Added a "Plaintext staging" section with what actually bounds the exposure (0700 root-owned, refuses a hijacked root), how long it lives, and the cleanup gaps: SIGKILL and power loss leave it, only the next BACKUP run sweeps, and the sweep registry is on tmpfs so a reboot loses the record. Threat model row corrected from "plaintext in memory". - --newkey was documented as the way to replace recipients "after a key compromise". It rewrites the recipient file only. The list is rebuilt from scratch before every backup, inline values first, so a compromised recipient in AGE_RECIPIENT comes straight back. Named every source the user must clear: AGE_RECIPIENT (which accumulates across lines), AGE_RECIPIENTS, the two environment variables, and the file itself, which is AGE_RECIPIENT_FILE and not necessarily the default path. - five snippets told the user to run age --decrypt against a bare .age file that does not exist: bundling is on by default and deletes the raw archive and its sidecars. These are the recovery instructions someone reads when ProxSave is gone, and none of them ran. Rewritten to start from the bundle, with the member names read out of tar -tf rather than typed, and a warning not to assume .tar.xz since the compressor falls back to gzip. - recipient.txt also carries the KDF salt as its leading comment, and the doc told the reader comments are ignored while listing neither the salt file under File Locations nor the salt in the migration row. Scoped the harm correctly, which the audit did not: decryption never reads these files, it takes the salt from the archive manifest, so existing backups are safe and only FUTURE ones become undecryptable from the passphrase. Added a recipe that rebuilds the salt from any archive. Also corrected in the same pass: AGE_RECIPIENT_FILE is "empty by default" only in code, the template sets it; and recipient.txt.bak-* is written by --newkey itself, not by the user. --- docs/ENCRYPTION.md | 246 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 211 insertions(+), 35 deletions(-) diff --git a/docs/ENCRYPTION.md b/docs/ENCRYPTION.md index c2212a4c..4e68311a 100644 --- a/docs/ENCRYPTION.md +++ b/docs/ENCRYPTION.md @@ -6,6 +6,7 @@ Complete guide to AGE encryption for Proxsave. - [Overview](#overview) - [Features](#features) +- [Plaintext staging](#plaintext-staging) - [Quick Start](#quick-start) - [Configure Recipients](#configure-recipients) - [Static Configuration](#static-configuration) @@ -25,7 +26,7 @@ Complete guide to AGE encryption for Proxsave. Proxsave uses the **[age](https://age-encryption.org/)** format (via `filippo.io/age`) for encryption. AGE is a modern, simple, and secure file encryption format designed to replace GPG for basic use cases. **Key characteristics**: -- **Streaming encryption**: Backups encrypted as they're created (no plaintext on disk) +- **Streaming encryption**: the archive is encrypted as it is written, so no plaintext archive is ever created on disk. The files gathered for the backup are staged in the clear under `/tmp/proxsave` first: see [Plaintext staging](#plaintext-staging) - **Multiple recipients**: Support for both passphrase and key-based encryption - **Memory safety**: Sensitive data zeroed immediately after use - **Standard format**: Compatible with standard AGE tools @@ -40,12 +41,55 @@ Proxsave uses the **[age](https://age-encryption.org/)** format (via `filippo.io | **Key types** | Passphrase or X25519 key pair. SSH public keys (`ssh-ed25519` / `ssh-rsa`) are accepted as recipients but ProxSave cannot decrypt with the matching SSH private key: see the warning below | | **Multiple recipients** | Single backup can be decrypted with any configured recipient | | **Interactive setup** | `--newkey` (or the first encrypted run) helps you configure recipients | -| **Streaming mode** | Encrypts during backup creation (no temporary plaintext) | +| **Streaming mode** | Encrypts during backup creation, so there is no temporary plaintext **archive**. The staging tree under `/tmp/proxsave` is plaintext | | **Security** | Passphrases read with `term.ReadPassword`, buffers zeroed after use | | **File permissions** | Recipient files are created 0700/0600; the security check verifies them and auto-fixes only when `AUTO_FIX_PERMISSIONS` is enabled (otherwise it warns) | --- +## Plaintext staging + +Encryption applies to the **archive**, not to the collection step. Each run creates a +staging directory `/tmp/proxsave/proxsave---` and copies every +collected file into it in the clear, including `/etc/shadow`, `/etc/gshadow`, +`/etc/ssl/private` and `/etc/pve/priv`. The tar is then read back from that directory and +streamed through the compressor into the age writer, so the archive is never written in +plaintext, but its input is. There is no tar file anywhere: the tar is streamed through a +pipe, not held in memory and not landed on disk. + +The staging root and the per-run directory are both created `0700` and owned by `root`, and +ProxSave refuses to run if `/tmp/proxsave` is a symlink, is not a directory, is group or +world writable, or is owned by another user. Staged files keep the owner and mode of their +originals, so the `0700` parent is what keeps other local users out. The location is not +configurable: it is compiled in, and `TMPDIR` does not move it. + +The directory exists from the start of collection until the run finishes, which includes +archiving, verification, bundling and any upload to secondary or cloud storage. It is +deleted when the run returns, whether it succeeded or failed, and also on Ctrl-C or +`SIGTERM`. It is **not** deleted if the process is killed with `SIGKILL` or the machine +loses power. The next *backup* run sweeps it (a restore or a status check does not), and +that sweep is driven by a registry, normally `/var/run/proxsave/temp-dirs.json` (it falls +back under `TMPDIR` when that directory cannot be created). On a stock host that path is +tmpfs, so a crash followed by a reboot loses the record and the leftover is never swept. +After an unclean shutdown, check by hand: + +```bash +ls -la /tmp/proxsave/ +rm -rf /tmp/proxsave/proxsave-* +``` + +Two practical consequences. `/tmp` needs room for a full uncompressed copy of everything +being backed up, on top of the archive itself. And if `/tmp` is a tmpfs the staged plaintext +is in RAM and can reach swap; if it is on disk, it is written to persistent storage. + +The same applies in reverse when you decrypt: `proxsave --decrypt` stages under +`/tmp/proxsave/proxmox-decrypt-*` and cleans up at the end, while a restore's safety backup +at `/tmp/proxsave/restore_backup_.tar.gz` is a plain unencrypted tar.gz that is +left in place deliberately. If you decrypt by hand with the `age` CLI, your own output is +plaintext too: pipe it rather than land it on a shared filesystem. + +--- + ## Quick Start ### 1. Generate Recipients @@ -108,7 +152,7 @@ Recipients are public keys or passphrases that can decrypt backups. A backup enc ### Static Configuration -**File**: set by `AGE_RECIPIENT_FILE`, which is **empty by default**; when unset ProxSave resolves it to `${BASE_DIR}/identity/age/recipient.txt`. +**File**: set by `AGE_RECIPIENT_FILE`. The shipped template sets it to `${BASE_DIR}/identity/age/recipient.txt`; the compiled-in default is empty, and ProxSave then resolves it to that same path. Check the key before assuming the default path: `--newkey` rewrites whatever `AGE_RECIPIENT_FILE` points at, not necessarily the default. ```plaintext # AGE recipients (one per line) @@ -128,6 +172,21 @@ ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIExampleSSHpublicKeyForAgeRecipient - Blank lines and `#` comments ignored - Supported types: X25519 (`age1...`) and SSH public keys (`ssh-ed25519` / `ssh-rsa`); mix freely +> **One comment in this file is not decoration.** ProxSave writes its own line at the top: +> +> ```plaintext +> # passphrase-salt: proxsave/age-passphrase/v2:1a2b3c... +> age1abc123def456ghi789jkl012mno345pqr678stu901vwx234yz567abc +> ``` +> +> The recipient parser ignores it, but ProxSave reads it back: it holds the +> per-installation salt that turns a passphrase into a recipient, and it takes priority +> over the `passphrase.salt` file beside it. Edit this file **in place** and leave the +> salt line alone. Do not retype the file from scratch and do not filter out comments. +> Losing both copies does not lock you out of existing backups (the salt travels in each +> archive's manifest) but every backup taken afterwards is written without a salt and can +> never be opened with the passphrase. + > **SSH keys encrypt, but ProxSave cannot decrypt with them.** `proxsave --decrypt` and `proxsave --restore` accept only an `AGE-SECRET-KEY-...` identity or a passphrase. Paste an SSH private key at the prompt and it is hashed as a passphrase, which derives the wrong identity and loops on "Provided key or passphrase does not match this archive." > > If you configure **only** SSH recipients, ProxSave cannot open its own archives. Always keep at least one `age1...` recipient or a passphrase alongside them. @@ -135,7 +194,9 @@ ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIExampleSSHpublicKeyForAgeRecipient > An archive encrypted to an SSH key can still be opened with the upstream `age` CLI, pointing `-i` at the SSH private key: > > ```bash -> age -d -i ~/.ssh/id_ed25519 -o backup.tar.xz backup.tar.xz.age +> # With bundling left at its default the raw .age is not on disk: untar the bundle first. +> tar -xf -backup-YYYYMMDD-HHMMSS.tar.xz.age.bundle.tar -C /tmp/emergency +> age -d -i ~/.ssh/id_ed25519 -o backup.tar.xz /tmp/emergency/-backup-YYYYMMDD-HHMMSS.tar.xz.age > ``` ### Interactive Wizard @@ -164,7 +225,7 @@ short list of well-known weak passphrases (for example `password`, `123456`, `qw is also rejected outright. **Notes**: -- Proxsave stores **only recipients** (public keys) in `${BASE_DIR}/identity/age/recipient.txt`. Keep private keys and passphrases offline. +- Proxsave stores **no private key and no passphrase**. Besides the recipients it does store the passphrase salt, a deliberately public value, in `identity/age/passphrase.salt` and as the `# passphrase-salt:` line inside the recipient file. Keep private keys and passphrases offline. - `AGE_RECIPIENT` (inline) and `AGE_RECIPIENT_FILE` are **merged and de-duplicated**. `AGE_RECIPIENTS` (plural) is accepted as a fallback alias for `AGE_RECIPIENT`, used only when `AGE_RECIPIENT` is empty. - Both TUI and CLI setup flows support multiple recipients and de-duplicate repeated entries before saving. @@ -189,17 +250,19 @@ proxsave ```text ┌─────────────────────────────────────────────┐ │ Phase 1: Backup Collection │ -│ - Gather PVE/PBS/System files │ -│ - Create TAR archive in memory │ +│ - Gather PVE/PBS/System files │ +│ - Stage them IN THE CLEAR under │ +│ /tmp/proxsave/proxsave--- │ +│ (mode 0700, root only) │ └─────────────┬───────────────────────────────┘ │ ▼ ┌─────────────────────────────────────────────┐ │ Phase 2: Streaming Encryption │ -│ - Read TAR stream │ -│ - Encrypt with AGE (ChaCha20-Poly1305) │ +│ - Stream the staging tree as a tar through │ +│ the compressor into AGE (ChaCha20-Poly1305)│ │ - Write to -backup-YYYYMMDD-HHMMSS.tar..age │ -│ - NO plaintext on disk │ +│ - NO plaintext ARCHIVE on disk │ └─────────────┬───────────────────────────────┘ │ ▼ @@ -261,12 +324,19 @@ proxsave --decrypt - the passphrase you used (proxsave derives the matching identity; the passphrase is not stored) **Output**: -- A decrypted bundle saved as: `*.decrypted.bundle.tar` +- A decrypted bundle saved as `*.decrypted.bundle.tar`. That is itself a plain tar holding + the decrypted archive plus its `.metadata` and `.sha256`, so `tar -xf` it to get the + archive out. -If you need fully scripted/non-interactive decryption with a **private key**, use the official `age` CLI tool: +If you need fully scripted/non-interactive decryption with a **private key**, use the +official `age` CLI. With bundling left at its default the raw `.age` is not on disk, so +unwrap the bundle first (see [Emergency Decryption Without +Configuration](#emergency-decryption-without-configuration)): ```bash -age --decrypt -i /path/to/age-keys.txt host-backup-YYYYMMDD-HHMMSS.tar.xz.age > host-backup-YYYYMMDD-HHMMSS.tar.xz +tar -xf -backup-YYYYMMDD-HHMMSS.tar.xz.age.bundle.tar -C /tmp/emergency +age --decrypt -i /path/to/age-keys.txt \ + /tmp/emergency/-backup-YYYYMMDD-HHMMSS.tar.xz.age > -backup-YYYYMMDD-HHMMSS.tar.xz ``` > **Passphrase recipients are not native age passphrases.** A passphrase recipient @@ -274,8 +344,16 @@ age --decrypt -i /path/to/age-keys.txt host-backup-YYYYMMDD-HHMMSS.tar.xz.age > > only understands age's own scrypt passphrase stanza) cannot decrypt it from the > passphrase alone, use `proxsave --decrypt`. proxsave re-derives the identity from > the passphrase plus the **per-installation random salt** generated at setup, which -> is stored next to the recipient (`identity/age/passphrase.salt`) and embedded in -> every backup manifest (`passphrase_salt`) so recovery works on any host. +> is stored next to the recipient (`identity/age/passphrase.salt`, mirrored as the +> `# passphrase-salt:` line in the recipient file) and embedded in every backup manifest +> (`passphrase_salt`) so recovery works on any host. The emergency `age` CLI path above +> therefore needs an `AGE-SECRET-KEY-...` identity file; a passphrase-only holder must use +> `proxsave --decrypt`, which reads the bundle directly. +> +> **Decryption never reads `recipient.txt` or `passphrase.salt`.** The salt travels inside +> each archive's manifest, so an existing backup stays openable with its passphrase even if +> both local copies are gone. See [Emergency +> Scenarios](#emergency-scenarios) for how to rebuild them from an archive. --- @@ -341,19 +419,46 @@ Rotating encryption keys periodically improves security (recommended annually or 4. After retention deletes older backups, remove the old recipient line from `${BASE_DIR}/identity/age/recipient.txt`. +5. If the old recipient was **also** set inline, remove it from `AGE_RECIPIENT` (or `AGE_RECIPIENTS`) and from the environment. Otherwise deleting the file line changes nothing: the inline value is merged back on the next backup. + **Important**: - Keep old private keys until you are sure all old backups are expired (or safely archived). - Proxsave stores only recipients; private keys/passphrases remain your responsibility. +- Before every backup ProxSave rebuilds the recipient list from scratch: inline values first, then the file's lines, then exact duplicates are dropped keeping the first occurrence. Nothing is remembered between runs, so any recipient still present in configuration comes back. ### Full Replacement (Reset Recipients) -To replace recipients completely (for example after a key compromise), run: +`proxsave --newkey` rewrites the **recipient file only**. It never edits +`configs/backup.env`. A recipient configured anywhere else stays active and is merged back +into the next backup, so on its own this is not a way to drop a compromised key. + +To really drop a compromised recipient, clear all of these: + +1. **`AGE_RECIPIENT` in `configs/backup.env`.** This key accumulates: every `AGE_RECIPIENT=` + line in the file is kept, and each line may hold several recipients separated by comma, + semicolon, pipe or newline. Remove or empty all of them. +2. **`AGE_RECIPIENTS`** (plural) in the same file. It is read only when `AGE_RECIPIENT` + yields nothing, so emptying `AGE_RECIPIENT` silently hands control to it. +3. **The `AGE_RECIPIENT` and `AGE_RECIPIENT_FILE` environment variables**, if a systemd + unit, cron wrapper or shell profile sets them. Environment values override the file. +4. **The recipient file itself**, which `--newkey` rewrites. That is the path in + `AGE_RECIPIENT_FILE`, which is only `${BASE_DIR}/identity/age/recipient.txt` when the key + is empty. + +Then run: ```bash proxsave --newkey ``` -This overwrites the recipient file after confirmation. Back up `${BASE_DIR}/identity/age/recipient.txt` first if you need rollback. +If the target recipient file already exists, `--newkey` asks for confirmation and copies the +old file to `.bak-` itself before overwriting. If it does not exist, for +example when your only recipient was inline, it writes the new file with no prompt and no +warning. + +The shipped template leaves `AGE_RECIPIENT` empty and nothing in the installer or the wizard +ever writes a value into it, so if you never hand-edited `backup.env` and set no environment +variable, `--newkey` does replace the effective recipient set. --- @@ -362,30 +467,97 @@ This overwrites the recipient file after confirmation. Back up `${BASE_DIR}/iden | Scenario | Solution | |----------|----------| | **Lost passphrase/private key** | **No recovery possible**. Keep 2+ offline copies (password manager, printed paper). | -| **Migrating to new server** | Copy the recipient file (`${BASE_DIR}/identity/age/recipient.txt`) and your `configs/backup.env`. Keep private keys offline. | +| **Migrating to new server** | Copy the whole `${BASE_DIR}/identity/age/` directory byte for byte (`recipient.txt` **and** `passphrase.salt`) plus your `configs/backup.env`. Do not retype the recipient file: the `# passphrase-salt:` line inside it is what lets a passphrase re-derive the key. If the salt does not reach the new host, backups taken there are written without a salt and can never be opened with the passphrase. Alternatively run `proxsave --newkey` on the new host and accept a new recipient. Keep private keys offline. | | **Verifying integrity** | Periodically decrypt a backup (or run a restore in a test VM) to ensure keys and archives are valid. | | **Automation** | Headless runs require recipients pre-configured (`AGE_RECIPIENT` and/or `AGE_RECIPIENT_FILE`). | -| **Recipient file overwritten** | Restore from your own backup copy (or from `recipient.txt.bak-*` if you created one). | +| **Recipient file overwritten** | Restore from `recipient.txt.bak-*`. ProxSave writes that copy itself whenever `--newkey` overwrites an existing recipient file. | +| **Passphrase salt lost** | Recover it from any archive's manifest: see [Rebuilding a lost salt](#rebuilding-a-lost-salt). Existing backups are unaffected. | ### Emergency Decryption Without Configuration -If you have the private key but lost all configuration: +If you have the private key but lost all configuration, start from what is actually on disk. +With bundling left at its default the **only** file on the backup path is +`-backup-YYYYMMDD-HHMMSS.tar..age.bundle.tar`. The raw `.age` archive, its +`.sha256`, its `.metadata` and its `.manifest.json` are deleted once the bundle is written, +and the bundle is also the only file copied to secondary and cloud destinations. Unwrap it +before anything else. ```bash -# Manual decryption with age tools -age --decrypt -i /path/to/age-keys.txt host-backup-YYYYMMDD-HHMMSS.tar.xz.age > decrypted.tar.xz +# 1. Read the member names from the bundle rather than typing them +tar -tf -backup-YYYYMMDD-HHMMSS.tar.xz.age.bundle.tar +# -backup-YYYYMMDD-HHMMSS.tar.xz.age.metadata +# -backup-YYYYMMDD-HHMMSS.tar.xz.age.sha256 +# -backup-YYYYMMDD-HHMMSS.tar.xz.age + +# 2. Unwrap. The bundle is an uncompressed tar with basename-only entries, +# so extract it into a directory of your own. +mkdir -p /tmp/emergency +tar -xf -backup-YYYYMMDD-HHMMSS.tar.xz.age.bundle.tar -C /tmp/emergency +cd /tmp/emergency + +# 3. Optional: verify before spending time on it. The .sha256 names the .age file, +# so run this from the extraction directory. +sha256sum -c -backup-YYYYMMDD-HHMMSS.tar.xz.age.sha256 + +# 4. Decrypt with the stock age CLI +age --decrypt -i /path/to/age-keys.txt \ + -backup-YYYYMMDD-HHMMSS.tar.xz.age > -backup-YYYYMMDD-HHMMSS.tar.xz + +# 5. Extract the inner archive +mkdir -p /tmp/emergency-restore +tar -xf -backup-YYYYMMDD-HHMMSS.tar.xz -C /tmp/emergency-restore +``` + +Notes on this recipe: + +- The `.age` file inside the bundle keeps exactly the name it had on disk, so step 4 is the + ordinary `age` command, just after the untar. +- **Do not assume the extension is `.tar.xz`.** It follows the compression actually used, and + ProxSave falls back to gzip when the configured compressor is not installed. Take the name + from `tar -tf`. +- If `BUNDLE_ASSOCIATED_FILES=false` was set there is no bundle: the `.age` file is already + on disk next to its sidecars, so skip steps 1 to 3. +- The `.metadata` member is a byte copy of the manifest JSON. Only `.metadata` is bundled, so + do not look for a `.manifest.json` inside the bundle. Read it with + `cat .age.metadata` to recover `compression_type`, `sha256`, `encryption_mode` and + `passphrase_salt` without touching the archive. +- This path needs an `AGE-SECRET-KEY-...` identity. A passphrase cannot be fed to the `age` + CLI (see the note under [Decrypting Backups](#decrypting-backups)); use + `proxsave --decrypt`, which reads the bundle directly. It only lists backups found under + the configured primary, secondary or cloud path, so a bundle carried in on removable media + has to be placed on one of those paths first. + +### Rebuilding a lost salt + +Decryption never reads `recipient.txt` or `passphrase.salt`, so an existing archive stays +openable even when both are gone. Use one to rebuild them. + +```bash +# Raw layout +grep passphrase_salt -backup-YYYYMMDD-HHMMSS.tar.xz.age.manifest.json + +# Bundled layout, where the manifest travels as the .metadata member +tar -xOf -backup-YYYYMMDD-HHMMSS.tar.xz.age.bundle.tar \ + -backup-YYYYMMDD-HHMMSS.tar.xz.age.metadata | grep passphrase_salt -# Extract archive -tar -xf decrypted.tar.xz -C /tmp/emergency-restore +# Write the value back, prefix included, and future backups record it again +printf '%s\n' 'proxsave/age-passphrase/v2:1a2b3c...' > ${BASE_DIR}/identity/age/passphrase.salt +chmod 600 ${BASE_DIR}/identity/age/passphrase.salt ``` +An archive whose manifest has no `passphrase_salt` at all was written either by a version +that used a fixed salt, which ProxSave still tries, or by an install that had already lost +the salt. For the second case there is no recovery. + ### Testing Backup Recoverability -Periodically verify backups are decryptable: +Periodically verify backups are decryptable. Reading the `.age` straight out of the bundle +avoids unpacking it: ```bash -# Example: decrypt with age CLI on a safe machine and list archive content -age --decrypt -i /path/to/age-keys.txt host-backup-YYYYMMDD-HHMMSS.tar.xz.age | tar -t >/dev/null && echo "✓ Archive valid" +tar -xOf -backup-YYYYMMDD-HHMMSS.tar.xz.age.bundle.tar \ + -backup-YYYYMMDD-HHMMSS.tar.xz.age \ + | age --decrypt -i /path/to/age-keys.txt | tar -t >/dev/null && echo "Archive valid" ``` **Recommended schedule**: Monthly automated test + manual review. @@ -397,7 +569,7 @@ age --decrypt -i /path/to/age-keys.txt host-backup-YYYYMMDD-HHMMSS.tar.xz.age | ### Encryption Implementation - **Algorithm**: ChaCha20-Poly1305 (AEAD) with X25519 ECDH -- **Key derivation**: scrypt (N=2^15, r=8, p=1) for passphrases. The current scheme uses a **per-installation random salt** (v2), generated once, stored `0600` at `identity/age/passphrase.salt`, and embedded in each manifest as `passphrase_salt` so the passphrase alone can re-derive the recipient on any host. At decrypt ProxSave tries salts in order: the manifest's per-install salt first, then two fixed legacy namespaces (`proxsave/age-passphrase/v1`, then the pre-rebrand `proxmox-backup-go/age-passphrase/v1`), so archives from older versions and from before the rename stay decryptable. +- **Key derivation**: scrypt (N=2^15, r=8, p=1) for passphrases. The current scheme uses a **per-installation random salt** (v2), generated once, stored `0600` at `identity/age/passphrase.salt`, mirrored as the `# passphrase-salt:` line inside the recipient file (that copy wins when both exist), and embedded in each manifest as `passphrase_salt` so the passphrase alone can re-derive the recipient on any host. At decrypt ProxSave tries salts in order: the manifest's per-install salt first, then two fixed legacy namespaces (`proxsave/age-passphrase/v1`, then the pre-rebrand `proxmox-backup-go/age-passphrase/v1`), so archives from older versions and from before the rename stay decryptable. - **Random nonces**: Unique per encryption operation - **Authentication**: Poly1305 MAC prevents tampering @@ -407,7 +579,7 @@ age --decrypt -i /path/to/age-keys.txt host-backup-YYYYMMDD-HHMMSS.tar.xz.age | |----------|----------------| | **Passphrase handling** | Read with `term.ReadPassword` (no echo) | | **Memory security** | Buffers zeroed immediately after use | -| **Streaming encryption** | No plaintext on disk during backup | +| **Streaming encryption** | No plaintext **archive** on disk. The staging tree is removed when the run ends | | **File permissions & ownership** | Enforced 0700/0600 and root:root on recipient/identity files (auto-fixed with `AUTO_FIX_PERMISSIONS`, otherwise warned) | | **Private key storage** | **Keep offline** (password manager, hardware token, printed backup) | | **Backup separation** | Store keys separately from backup media | @@ -454,7 +626,7 @@ age --decrypt -i /path/to/age-keys.txt host-backup-YYYYMMDD-HHMMSS.tar.xz.age | - ✅ Network interception (if using rclone with encryption) **Not protected against**: -- ❌ Compromise of the server during backup (plaintext in memory) +- ❌ Compromise of the server during a backup. While a backup runs, the collected files sit unencrypted under `/tmp/proxsave` (see [Plaintext staging](#plaintext-staging)); a root-level compromise in that window sees everything in the clear - ❌ Private key theft from offline storage - ❌ Weak passphrase brute-force - ❌ Advanced persistent threats on backup server @@ -508,7 +680,8 @@ AGE encryption meets requirements for: ENCRYPT_ARCHIVE=true # Master switch # Recipient configuration -# AGE_RECIPIENT_FILE is empty by default; when unset it resolves to ${BASE_DIR}/identity/age/recipient.txt +# The shipped template sets this; the compiled-in default is empty and resolves to the same path. +# --newkey rewrites whatever this points at, so check it before assuming the default. AGE_RECIPIENT_FILE=${BASE_DIR}/identity/age/recipient.txt # Public recipients (recommended) # Optional: inline recipients (merged with file; supports comma/semicolon/pipe/newline) @@ -531,8 +704,10 @@ proxsave --decrypt # Restore from encrypted backup proxsave --restore -# Manual decryption (scriptable) with age CLI -age --decrypt -i /path/to/age-keys.txt host-backup-YYYYMMDD-HHMMSS.tar.xz.age > host-backup-YYYYMMDD-HHMMSS.tar.xz +# Manual decryption (scriptable) with age CLI, straight out of the bundle +tar -xOf -backup-YYYYMMDD-HHMMSS.tar.xz.age.bundle.tar \ + -backup-YYYYMMDD-HHMMSS.tar.xz.age \ + | age --decrypt -i /path/to/age-keys.txt > -backup-YYYYMMDD-HHMMSS.tar.xz ``` ### File Locations @@ -543,8 +718,9 @@ configs/ identity/ └── age/ - ├── recipient.txt # Public recipients (0600) - └── recipient.txt.bak-* # Optional backups (if you made one) + ├── recipient.txt # Public recipients, plus the "# passphrase-salt:" line (0600) + ├── passphrase.salt # Per-installation passphrase salt (0600, passphrase setups only) + └── recipient.txt.bak-* # Written by --newkey when it overwrites an existing file backup/ └── -backup-*.tar.[.age][.bundle.tar] From fd69d3b5994eac040f0eac07b953ce9c40da213b Mon Sep 17 00:00:00 2001 From: tis24dev Date: Sun, 2 Aug 2026 22:51:48 +0200 Subject: [PATCH 06/50] docs(encryption): fix what the refuters broke, including my own inverted salt precedence Three lenses, all BREAK, 22 findings. The worst were mine, not the audit's. - I inverted the salt precedence. I wrote that the "# passphrase-salt:" comment in recipient.txt "takes priority over passphrase.salt". The opposite is true in effect: every backup calls backfillCoLocatedPassphraseSalt() before the manifest salt is read, and that rewrites the comment FROM the sibling, so the sibling wins whenever the two differ. The comment is only consulted once the sibling is gone. - Worse, I wrote that the comment "is what lets a passphrase re-derive the key" in the migration row. It is not. getOrCreatePassphraseSalt reads only passphrase.salt and, when that file is absent, mints a NEW random salt and derives a DIFFERENT recipient, without consulting the comment and without warning. Telling a migrating user the comment was enough was exactly the data loss this section exists to prevent. - The recoverability test I added cannot succeed. tar auto-detects compression only for a named file; piping an xz stream into tar -t dies with "Archive is compressed. Use -J option", exit 2, so the && echo never fires. Measured, not reasoned about. Rewritten to land the plaintext and list it from the file, which stays correct for xz, gzip and zstd alike. - The staging section claimed the root and the per-run directory are both 0700. Only the per-run one is: /tmp/proxsave is created 0755 by the pre-backup check and by the safety-backup path, and the guard refuses only a group/world WRITABLE root, so a world-readable root is accepted and never tightened. That matters because the restore safety tarballs are written 0644 straight into that root, and they contain /etc/shadow and /etc/pve/priv material whenever those categories were restored. - The section listed the leftovers incompletely. A restore extracts the sensitive categories in the clear to /tmp/proxsave/restore-stage-*/ and NOTHING ever deletes it, on success or failure; it is not registered so the sweep cannot reach it either. The cleanup command I gave matched only backup staging, so a reader following it to purge plaintext left the decrypted shadow and pve priv sitting in /tmp believing they were done. Smaller ones, all verified: a second Ctrl-C skips every deferred cleanup because the signal handler is one-shot; ${BASE_DIR} is not a shell variable, so the salt-recovery command wrote to /identity/...; two untar snippets were missing their mkdir; the Key Rotation section still carried the "stores only recipients" absolute this commit series removed elsewhere, and still pointed at the default recipient path rather than AGE_RECIPIENT_FILE; the manifest sample had no passphrase_salt field though the new text sends the reader to grep for it; and the registry path is overridable by PROXMOX_TEMP_REGISTRY_PATH. --- docs/ENCRYPTION.md | 134 +++++++++++++++++++++++++++++++-------------- 1 file changed, 94 insertions(+), 40 deletions(-) diff --git a/docs/ENCRYPTION.md b/docs/ENCRYPTION.md index 4e68311a..07b9b7f9 100644 --- a/docs/ENCRYPTION.md +++ b/docs/ENCRYPTION.md @@ -57,36 +57,58 @@ streamed through the compressor into the age writer, so the archive is never wri plaintext, but its input is. There is no tar file anywhere: the tar is streamed through a pipe, not held in memory and not landed on disk. -The staging root and the per-run directory are both created `0700` and owned by `root`, and -ProxSave refuses to run if `/tmp/proxsave` is a symlink, is not a directory, is group or -world writable, or is owned by another user. Staged files keep the owner and mode of their -originals, so the `0700` parent is what keeps other local users out. The location is not -configurable: it is compiled in, and `TMPDIR` does not move it. +The **per-run** directory is created `0700` and owned by `root`, and that is what keeps other +local users out of the staged files, since staged files keep the owner and mode of their +originals. The shared root `/tmp/proxsave` is a different matter: several paths create it +`0755`, and the guard only refuses a symlink, a non-directory, a **group or world writable** +root, or one owned by another user. A world-readable `0755` root is accepted and never +tightened, so anything ProxSave writes directly into the root, rather than inside a `0700` +per-run directory, is readable by every local user. Run `chmod 700 /tmp/proxsave` if that +matters on your host. The location is not configurable: it is compiled in, and `TMPDIR` does +not move it. The directory exists from the start of collection until the run finishes, which includes archiving, verification, bundling and any upload to secondary or cloud storage. It is -deleted when the run returns, whether it succeeded or failed, and also on Ctrl-C or -`SIGTERM`. It is **not** deleted if the process is killed with `SIGKILL` or the machine -loses power. The next *backup* run sweeps it (a restore or a status check does not), and -that sweep is driven by a registry, normally `/var/run/proxsave/temp-dirs.json` (it falls -back under `TMPDIR` when that directory cannot be created). On a stock host that path is -tmpfs, so a crash followed by a reboot loses the record and the leftover is never swept. -After an unclean shutdown, check by hand: +deleted when the run returns, whether it succeeded or failed, and also on the **first** +Ctrl-C or `SIGTERM`. It is **not** deleted if the process is killed with `SIGKILL`, if the +machine loses power, or if you press **Ctrl-C a second time**: ProxSave un-registers its +signal handler after the first signal, so a second one terminates the process outright and +no cleanup runs. Give the first Ctrl-C time to unwind. + +The next *backup* run sweeps leftovers (a restore or a status check does not), driven by a +registry at `/var/run/proxsave/temp-dirs.json`, overridden by `PROXMOX_TEMP_REGISTRY_PATH` +when set and falling back under `TMPDIR` when that directory cannot be created. An entry is +swept when its PID is gone, or unconditionally once the record is 24 hours old. On a stock +host `/var/run` is tmpfs, so a crash followed by a reboot loses the record and the leftover +is never swept. Check by hand after an unclean shutdown, and note that the sweep only ever +covers backup staging: ```bash ls -la /tmp/proxsave/ -rm -rf /tmp/proxsave/proxsave-* +rm -rf /tmp/proxsave/proxsave-* # backup staging, from a killed backup +rm -rf /tmp/proxsave/proxmox-decrypt-* # decrypt staging: a FULLY DECRYPTED archive +rm -rf /tmp/proxsave/restore-stage-* # restore staging: plaintext shadow and pve priv +rm -f /tmp/proxsave/*_backup_*.tar.gz # restore safety tarballs, once the restore is settled ``` Two practical consequences. `/tmp` needs room for a full uncompressed copy of everything being backed up, on top of the archive itself. And if `/tmp` is a tmpfs the staged plaintext is in RAM and can reach swap; if it is on disk, it is written to persistent storage. -The same applies in reverse when you decrypt: `proxsave --decrypt` stages under -`/tmp/proxsave/proxmox-decrypt-*` and cleans up at the end, while a restore's safety backup -at `/tmp/proxsave/restore_backup_.tar.gz` is a plain unencrypted tar.gz that is -left in place deliberately. If you decrypt by hand with the `age` CLI, your own output is -plaintext too: pipe it rather than land it on a shared filesystem. +The same applies in reverse, and the restore side is worse. `proxsave --decrypt` stages under +`/tmp/proxsave/proxmox-decrypt-*` and removes it at the end of a normal run. `proxsave +--restore` extracts the sensitive categories in the clear into +`/tmp/proxsave/restore-stage-_/`, which holds material such as `/etc/shadow`, +`/etc/gshadow` and `/etc/pve/priv/*.cfg`, and **never deletes it**, on success or failure. +Nothing sweeps it either, since it is not registered. A restore also leaves its rollback and +safety tarballs (`restore_backup_`, `network_rollback_backup_`, `firewall_rollback_backup_`, +`ha_rollback_backup_`, `pve_access_control_rollback_backup_`, each `_.tar.gz`) +deliberately in place, and those are written **mode 0644 directly in the root**, not inside a +`0700` directory, so on a stock host any local user can read them. Clean them up yourself +once a restore has settled. + +If you decrypt by hand with the `age` CLI, your own output is plaintext too: pipe it rather +than land it on a shared filesystem. --- @@ -179,13 +201,20 @@ ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIExampleSSHpublicKeyForAgeRecipient > age1abc123def456ghi789jkl012mno345pqr678stu901vwx234yz567abc > ``` > -> The recipient parser ignores it, but ProxSave reads it back: it holds the -> per-installation salt that turns a passphrase into a recipient, and it takes priority -> over the `passphrase.salt` file beside it. Edit this file **in place** and leave the -> salt line alone. Do not retype the file from scratch and do not filter out comments. -> Losing both copies does not lock you out of existing backups (the salt travels in each -> archive's manifest) but every backup taken afterwards is written without a salt and can -> never be opened with the passphrase. +> The recipient parser ignores it, but ProxSave reads it back. It is a **copy** of the +> per-installation salt that gets stamped into every archive manifest. +> `identity/age/passphrase.salt` is the authoritative one: it is the only copy the setup +> wizard reads when it derives a recipient from a passphrase, and every backup rewrites +> this comment from it. The comment exists so that losing the sibling does not lose the +> salt for future manifests. +> +> Edit this file **in place** and leave the salt line alone. Do not retype the file from +> scratch and do not filter out comments. Losing both copies does not lock you out of +> existing backups, since the salt travels in each archive's manifest, but every backup +> taken afterwards is written without a salt and can never be opened with the passphrase. +> And if `passphrase.salt` is gone, re-running the wizard with the same passphrase mints a +> **new random salt** and derives a **different** recipient, without a warning: the comment +> is not consulted there. > **SSH keys encrypt, but ProxSave cannot decrypt with them.** `proxsave --decrypt` and `proxsave --restore` accept only an `AGE-SECRET-KEY-...` identity or a passphrase. Paste an SSH private key at the prompt and it is hashed as a passphrase, which derives the wrong identity and loops on "Provided key or passphrase does not match this archive." > @@ -195,6 +224,7 @@ ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIExampleSSHpublicKeyForAgeRecipient > > ```bash > # With bundling left at its default the raw .age is not on disk: untar the bundle first. +> mkdir -p /tmp/emergency > tar -xf -backup-YYYYMMDD-HHMMSS.tar.xz.age.bundle.tar -C /tmp/emergency > age -d -i ~/.ssh/id_ed25519 -o backup.tar.xz /tmp/emergency/-backup-YYYYMMDD-HHMMSS.tar.xz.age > ``` @@ -301,10 +331,14 @@ backup/ "created_at": "2024-01-15T02:30:00Z", "compression_type": "xz", "hostname": "pve-node1", - "encryption_mode": "age" + "encryption_mode": "age", + "passphrase_salt": "proxsave/age-passphrase/v2:1a2b3c..." } ``` +`passphrase_salt` is omitted entirely for X25519 or SSH-only setups, and for legacy +fixed-salt archives. + --- ## Decrypting Backups @@ -334,6 +368,7 @@ unwrap the bundle first (see [Emergency Decryption Without Configuration](#emergency-decryption-without-configuration)): ```bash +mkdir -p /tmp/emergency tar -xf -backup-YYYYMMDD-HHMMSS.tar.xz.age.bundle.tar -C /tmp/emergency age --decrypt -i /path/to/age-keys.txt \ /tmp/emergency/-backup-YYYYMMDD-HHMMSS.tar.xz.age > -backup-YYYYMMDD-HHMMSS.tar.xz @@ -410,20 +445,23 @@ Rotating encryption keys periodically improves security (recommended annually or age-keygen -o age-keys-2025.txt ``` -2. Extract the new public recipient and append it to the recipient file: +2. Extract the new public recipient and append it to the file named by `AGE_RECIPIENT_FILE` + (that is `${BASE_DIR}/identity/age/recipient.txt` only when the key is empty). ProxSave + reads exactly one recipient file, so appending to the default path while the key points + elsewhere is a silent no-op and the new key never enters the rotation: ```bash - grep "# public key:" age-keys-2025.txt | cut -d: -f2 | tr -d ' ' >> ${BASE_DIR}/identity/age/recipient.txt + grep "# public key:" age-keys-2025.txt | cut -d: -f2 | tr -d ' ' >> /opt/proxsave/identity/age/recipient.txt ``` 3. Run backups for a while: new backups can be decrypted with **either** the old or the new private key. -4. After retention deletes older backups, remove the old recipient line from `${BASE_DIR}/identity/age/recipient.txt`. +4. After retention deletes older backups, remove the old recipient line from that same file. 5. If the old recipient was **also** set inline, remove it from `AGE_RECIPIENT` (or `AGE_RECIPIENTS`) and from the environment. Otherwise deleting the file line changes nothing: the inline value is merged back on the next backup. **Important**: - Keep old private keys until you are sure all old backups are expired (or safely archived). -- Proxsave stores only recipients; private keys/passphrases remain your responsibility. +- Proxsave stores no private key and no passphrase. Besides the recipients it stores the passphrase salt (`identity/age/passphrase.salt` and the `# passphrase-salt:` line in the recipient file). Private keys and passphrases remain your responsibility. - Before every backup ProxSave rebuilds the recipient list from scratch: inline values first, then the file's lines, then exact duplicates are dropped keeping the first occurrence. Nothing is remembered between runs, so any recipient still present in configuration comes back. ### Full Replacement (Reset Recipients) @@ -467,11 +505,11 @@ variable, `--newkey` does replace the effective recipient set. | Scenario | Solution | |----------|----------| | **Lost passphrase/private key** | **No recovery possible**. Keep 2+ offline copies (password manager, printed paper). | -| **Migrating to new server** | Copy the whole `${BASE_DIR}/identity/age/` directory byte for byte (`recipient.txt` **and** `passphrase.salt`) plus your `configs/backup.env`. Do not retype the recipient file: the `# passphrase-salt:` line inside it is what lets a passphrase re-derive the key. If the salt does not reach the new host, backups taken there are written without a salt and can never be opened with the passphrase. Alternatively run `proxsave --newkey` on the new host and accept a new recipient. Keep private keys offline. | +| **Migrating to new server** | Copy the whole `identity/age/` directory byte for byte, **`passphrase.salt` included**, plus your `configs/backup.env`. `passphrase.salt` is the file that matters: the setup wizard reads only that one, and if it is missing it mints a new random salt and derives a **different** recipient without warning. The `# passphrase-salt:` copy inside `recipient.txt` only feeds the manifest, so do not rely on it alone and do not retype the recipient file. Alternatively run `proxsave --newkey` on the new host and accept a new recipient. Keep private keys offline. | | **Verifying integrity** | Periodically decrypt a backup (or run a restore in a test VM) to ensure keys and archives are valid. | | **Automation** | Headless runs require recipients pre-configured (`AGE_RECIPIENT` and/or `AGE_RECIPIENT_FILE`). | | **Recipient file overwritten** | Restore from `recipient.txt.bak-*`. ProxSave writes that copy itself whenever `--newkey` overwrites an existing recipient file. | -| **Passphrase salt lost** | Recover it from any archive's manifest: see [Rebuilding a lost salt](#rebuilding-a-lost-salt). Existing backups are unaffected. | +| **Passphrase salt lost** | Recover it from the manifest of an archive your current recipient still opens: see [Rebuilding a lost salt](#rebuilding-a-lost-salt). Existing backups are unaffected. Write back only a value that matches the recipient you are still using. | ### Emergency Decryption Without Configuration @@ -540,24 +578,35 @@ grep passphrase_salt -backup-YYYYMMDD-HHMMSS.tar.xz.age.manifest.json tar -xOf -backup-YYYYMMDD-HHMMSS.tar.xz.age.bundle.tar \ -backup-YYYYMMDD-HHMMSS.tar.xz.age.metadata | grep passphrase_salt -# Write the value back, prefix included, and future backups record it again -printf '%s\n' 'proxsave/age-passphrase/v2:1a2b3c...' > ${BASE_DIR}/identity/age/passphrase.salt -chmod 600 ${BASE_DIR}/identity/age/passphrase.salt +# Write the value back, prefix included, and future backups record it again. +# BASE_DIR is not a shell variable: substitute your install root. +printf '%s\n' 'proxsave/age-passphrase/v2:1a2b3c...' > /opt/proxsave/identity/age/passphrase.salt +chmod 600 /opt/proxsave/identity/age/passphrase.salt ``` +Write back only a salt that matches the recipient you are still using. The next backup copies +this file over the `# passphrase-salt:` line in `recipient.txt` and stamps it into every new +manifest, so a value from an older salt generation would make future archives unopenable with +the passphrase you have. + An archive whose manifest has no `passphrase_salt` at all was written either by a version that used a fixed salt, which ProxSave still tries, or by an install that had already lost the salt. For the second case there is no recovery. ### Testing Backup Recoverability -Periodically verify backups are decryptable. Reading the `.age` straight out of the bundle -avoids unpacking it: +Periodically verify backups are decryptable. Land the plaintext first: `tar` auto-detects +compression only when it is given a **file name**, so piping a compressed stream into +`tar -t` fails with `Archive is compressed. Use -J option` no matter which compressor was +used. ```bash +mkdir -p /tmp/emergency tar -xOf -backup-YYYYMMDD-HHMMSS.tar.xz.age.bundle.tar \ -backup-YYYYMMDD-HHMMSS.tar.xz.age \ - | age --decrypt -i /path/to/age-keys.txt | tar -t >/dev/null && echo "Archive valid" + | age --decrypt -i /path/to/age-keys.txt > /tmp/emergency/archive.inner +tar -tf /tmp/emergency/archive.inner >/dev/null && echo "Archive valid" +rm -f /tmp/emergency/archive.inner ``` **Recommended schedule**: Monthly automated test + manual review. @@ -569,7 +618,7 @@ tar -xOf -backup-YYYYMMDD-HHMMSS.tar.xz.age.bundle.tar \ ### Encryption Implementation - **Algorithm**: ChaCha20-Poly1305 (AEAD) with X25519 ECDH -- **Key derivation**: scrypt (N=2^15, r=8, p=1) for passphrases. The current scheme uses a **per-installation random salt** (v2), generated once, stored `0600` at `identity/age/passphrase.salt`, mirrored as the `# passphrase-salt:` line inside the recipient file (that copy wins when both exist), and embedded in each manifest as `passphrase_salt` so the passphrase alone can re-derive the recipient on any host. At decrypt ProxSave tries salts in order: the manifest's per-install salt first, then two fixed legacy namespaces (`proxsave/age-passphrase/v1`, then the pre-rebrand `proxmox-backup-go/age-passphrase/v1`), so archives from older versions and from before the rename stay decryptable. +- **Key derivation**: scrypt (N=2^15, r=8, p=1) for passphrases. The current scheme uses a **per-installation random salt** (v2), generated once, stored `0600` at `identity/age/passphrase.salt`, mirrored as the `# passphrase-salt:` line inside the recipient file (every backup rewrites that comment from the sibling, so the sibling wins whenever the two differ; the comment is only consulted once the sibling is gone), and embedded in each manifest as `passphrase_salt` so the passphrase alone can re-derive the recipient on any host. At decrypt ProxSave tries salts in order: the manifest's per-install salt first, then two fixed legacy namespaces (`proxsave/age-passphrase/v1`, then the pre-rebrand `proxmox-backup-go/age-passphrase/v1`), so archives from older versions and from before the rename stay decryptable. - **Random nonces**: Unique per encryption operation - **Authentication**: Poly1305 MAC prevents tampering @@ -579,7 +628,7 @@ tar -xOf -backup-YYYYMMDD-HHMMSS.tar.xz.age.bundle.tar \ |----------|----------------| | **Passphrase handling** | Read with `term.ReadPassword` (no echo) | | **Memory security** | Buffers zeroed immediately after use | -| **Streaming encryption** | No plaintext **archive** on disk. The staging tree is removed when the run ends | +| **Streaming encryption** | No plaintext **archive** on disk. The backup staging tree is removed when the run ends, but not on `SIGKILL`, power loss or a double Ctrl-C, and a restore leaves its own staging tree and safety tarballs behind: see [Plaintext staging](#plaintext-staging) | | **File permissions & ownership** | Enforced 0700/0600 and root:root on recipient/identity files (auto-fixed with `AUTO_FIX_PERMISSIONS`, otherwise warned) | | **Private key storage** | **Keep offline** (password manager, hardware token, printed backup) | | **Backup separation** | Store keys separately from backup media | @@ -710,6 +759,11 @@ tar -xOf -backup-YYYYMMDD-HHMMSS.tar.xz.age.bundle.tar \ | age --decrypt -i /path/to/age-keys.txt > -backup-YYYYMMDD-HHMMSS.tar.xz ``` +### File paths in these examples + +`BASE_DIR` is auto-detected from the installed executable and is **not** a shell variable: +substitute your install root (typically `/opt/proxsave`) when pasting any path that uses it. + ### File Locations ```text From aedbe0c9436e8b2e77fb1367d2170b9974956396 Mon Sep 17 00:00:00 2001 From: tis24dev Date: Sun, 2 Aug 2026 23:23:55 +0200 Subject: [PATCH 07/50] refactor(install): share the existing-config decision and the cron run-time seed The "backup.env already exists" question was answered twice. cmd/proxsave had its own existingConfigMode enum, its own decision struct and its own resolver; the Charm screen used installer.ExistingConfigAction with the equivalent switch inlined in install_tui.go. The os.Stat/IsNotExist/IsRegular pre-check was a verbatim third copy. All of it now lives in internal/installer/existing_config.go and both front-ends keep only their prompting. The shared decision carries the RAW base template, where "" means "use the embedded default". That emptiness is load-bearing: ApplyInstallData derives editingExisting from it, so handing it an expanded default on Overwrite would silently change which keys are treated as pre-existing. The CLI wizard computes its own prompt defaults from the base, so it expands via BaseTemplateOrDefault -- but only off the Edit path, where a blank backup.env must stay blank. The cron run-time seed unifies on the CLI's gate (Edit only). The TUI used to derive on Keep-existing too, log the adoption note during the interactive phase and then log it again at the commit point, while never reading the early value. Keep still gets its SCHEDULER_TIME at the unchanged commit point; the note is now logged once. Two bugs fixed along the way. The TUI's mirror had no non-blank guard, so editing a blank backup.env turned the base into "\nSCHEDULER_TIME=HH:MM", defeated ApplyInstallData's blank->default substitution and wrote a gutted config. And the adoption note was logged before the mirror ran, so on that same blank base it promised "the daily run time does not change" and then discarded the value; it is now emitted only when the seed actually lands. Both new gates are pinned by tests that were verified to fail without them: the crontab derive is counted per answer, and a blank-base Edit asserts the template comes back raw. Neither was covered before -- removing either gate left the whole suite, characterization goldens included, green. Not delivered here: aligning the CLI to the TUI on a blank backup.env + Edit. The empty-base branch in schedulerEngineDefault/healthcheckModeDefault/cronTimeDefault is dead at runtime because configureSecondaryStorage always mutates the template first, so that alignment needs the prompt defaults computed from the raw base -- a separate change. The doc comments claiming that boundary matches Charm are false today and are left for it. --- cmd/proxsave/install.go | 21 +-- cmd/proxsave/install_existing_config.go | 105 +++---------- cmd/proxsave/install_existing_config_test.go | 96 ++---------- cmd/proxsave/install_test.go | 38 +++++ cmd/proxsave/install_tui.go | 39 ++--- cmd/proxsave/schedule_helpers.go | 44 +++--- cmd/proxsave/scheduler_time_seed_test.go | 105 +++++++++++-- internal/installer/existing_config.go | 148 ++++++++++++++++++ internal/installer/existing_config_test.go | 156 +++++++++++++++++++ internal/installer/ui_support.go | 11 -- internal/ui/flows/install/install.go | 12 +- 11 files changed, 524 insertions(+), 251 deletions(-) create mode 100644 internal/installer/existing_config.go create mode 100644 internal/installer/existing_config_test.go diff --git a/cmd/proxsave/install.go b/cmd/proxsave/install.go index 68cd910b..5aa408ba 100644 --- a/cmd/proxsave/install.go +++ b/cmd/proxsave/install.go @@ -748,19 +748,22 @@ func prepareBaseTemplate(ctx context.Context, reader *bufio.Reader, configPath s // "Run at" prompt. Cancelling anywhere later then leaves the host byte-identical, // which a write at this point would not. Keep existing has no wizard to carry the // value, so its write is deferred to the commit point in runInstall. - if decision.FromExistingFile { - if seed := deriveSchedulerTimeFromCrontabFn(ctx, configPath); seed.Note != "" { - logBootstrapInfo(bootstrap, "%s", seed.Note) - if seed.Time != "" && decision.BaseTemplate != "" { - decision.BaseTemplate = setEnvValue(decision.BaseTemplate, "SCHEDULER_TIME", seed.Time) - } - } - } + decision.BaseTemplate = adoptCronRunTimeIntoBase(ctx, decision, configPath, bootstrap) if decision.SkipConfigWizard { fmt.Println("Existing configuration detected, keeping current backup.env and skipping configuration wizard.") return "", true, false, nil } - return decision.BaseTemplate, false, decision.FromExistingFile, nil + // The shared decision carries the RAW base ("" = embedded default) because + // ApplyInstallData derives editingExisting from it. The CLI wizard, unlike the + // Charm one, computes its own prompt defaults from this template, so it expands + // here - but ONLY off the Edit path: expanding a blank existing backup.env would + // rewrite it as the full embedded template instead of the minimal key set it + // produces today. Pinned by TestPrepareBaseTemplateEditBlankBaseStaysRaw. + base := decision.BaseTemplate + if !decision.FromExistingFile { + base = installer.BaseTemplateOrDefault(base) + } + return base, false, decision.FromExistingFile, nil } func configureSecondaryStorage(ctx context.Context, reader *bufio.Reader, template string) (string, error) { diff --git a/cmd/proxsave/install_existing_config.go b/cmd/proxsave/install_existing_config.go index 1d661105..985e90bc 100644 --- a/cmd/proxsave/install_existing_config.go +++ b/cmd/proxsave/install_existing_config.go @@ -4,50 +4,25 @@ import ( "bufio" "context" "fmt" - "os" "strings" - "github.com/tis24dev/proxsave/internal/config" - "github.com/tis24dev/proxsave/internal/safefs" + "github.com/tis24dev/proxsave/internal/installer" ) -type existingConfigMode int - -const ( - existingConfigOverwrite existingConfigMode = iota - existingConfigEdit - existingConfigKeepContinue - existingConfigCancel -) - -type existingConfigDecision struct { - BaseTemplate string - SkipConfigWizard bool - AbortInstall bool - // FromExistingFile is true only when the wizard starts from the user's current - // backup.env (Edit). Fresh installs and Overwrite start from the embedded - // template, so defaults (e.g. the scheduler engine) may be the recommended new - // values rather than the stored ones. - FromExistingFile bool -} - -func promptExistingConfigModeCLI(ctx context.Context, reader *bufio.Reader, configPath string) (existingConfigMode, error) { +func promptExistingConfigModeCLI(ctx context.Context, reader *bufio.Reader, configPath string) (installer.ExistingConfigAction, error) { if ctx == nil { ctx = context.Background() } - info, err := os.Stat(configPath) + exists, err := installer.ExistingConfigPresent(configPath) if err != nil { - if os.IsNotExist(err) { - if err := ctx.Err(); err != nil { - return existingConfigCancel, err - } - return existingConfigOverwrite, nil - } - return existingConfigCancel, fmt.Errorf("failed to access configuration file: %w", err) + return installer.ExistingConfigCancel, err } - if !info.Mode().IsRegular() { - return existingConfigCancel, fmt.Errorf("configuration file path is not a regular file: %s", configPath) + if !exists { + if err := ctx.Err(); err != nil { + return installer.ExistingConfigCancel, err + } + return installer.ExistingConfigOverwrite, nil } fmt.Printf("%s already exists.\n", configPath) @@ -60,77 +35,41 @@ func promptExistingConfigModeCLI(ctx context.Context, reader *bufio.Reader, conf for { choice, err := promptOptional(ctx, reader, "Choice [3]: ") if err != nil { - return existingConfigCancel, err + return installer.ExistingConfigCancel, err } switch strings.TrimSpace(choice) { case "": fallthrough case "3": if err := ctx.Err(); err != nil { - return existingConfigCancel, err + return installer.ExistingConfigCancel, err } - return existingConfigKeepContinue, nil + return installer.ExistingConfigKeepContinue, nil case "1": if err := ctx.Err(); err != nil { - return existingConfigCancel, err + return installer.ExistingConfigCancel, err } - return existingConfigOverwrite, nil + return installer.ExistingConfigOverwrite, nil case "2": if err := ctx.Err(); err != nil { - return existingConfigCancel, err + return installer.ExistingConfigCancel, err } - return existingConfigEdit, nil + return installer.ExistingConfigEdit, nil case "0": if err := ctx.Err(); err != nil { - return existingConfigCancel, err + return installer.ExistingConfigCancel, err } - return existingConfigCancel, nil + return installer.ExistingConfigCancel, nil default: fmt.Println("Please enter 1, 2, 3 or 0.") } } } -func resolveExistingConfigDecision(mode existingConfigMode, configPath string) (existingConfigDecision, error) { - switch mode { - case existingConfigOverwrite: - return existingConfigDecision{ - BaseTemplate: config.DefaultEnvTemplate(), - SkipConfigWizard: false, - AbortInstall: false, - }, nil - case existingConfigEdit: - content, err := safefs.ReadFileUnderRoot(configPath) - if err != nil { - return existingConfigDecision{}, fmt.Errorf("read existing configuration: %w", err) - } - return existingConfigDecision{ - BaseTemplate: string(content), - SkipConfigWizard: false, - AbortInstall: false, - FromExistingFile: true, - }, nil - case existingConfigKeepContinue: - return existingConfigDecision{ - BaseTemplate: "", - SkipConfigWizard: true, - AbortInstall: false, - }, nil - case existingConfigCancel: - return existingConfigDecision{ - BaseTemplate: "", - SkipConfigWizard: false, - AbortInstall: true, - }, nil - default: - return existingConfigDecision{}, fmt.Errorf("unsupported existing configuration mode: %d", mode) - } -} - -func prepareExistingConfigDecisionCLI(ctx context.Context, reader *bufio.Reader, configPath string) (existingConfigDecision, error) { - mode, err := promptExistingConfigModeCLI(ctx, reader, configPath) +func prepareExistingConfigDecisionCLI(ctx context.Context, reader *bufio.Reader, configPath string) (installer.ExistingConfigDecision, error) { + action, err := promptExistingConfigModeCLI(ctx, reader, configPath) if err != nil { - return existingConfigDecision{}, err + return installer.ExistingConfigDecision{}, err } - return resolveExistingConfigDecision(mode, configPath) + return installer.ResolveExistingConfigDecision(action, configPath) } diff --git a/cmd/proxsave/install_existing_config_test.go b/cmd/proxsave/install_existing_config_test.go index dd5f358e..ded73caf 100644 --- a/cmd/proxsave/install_existing_config_test.go +++ b/cmd/proxsave/install_existing_config_test.go @@ -4,10 +4,11 @@ import ( "bufio" "context" "errors" - "os" "path/filepath" "strings" "testing" + + "github.com/tis24dev/proxsave/internal/installer" ) func TestPromptExistingConfigModeCLIMissingFileDefaultsToOverwrite(t *testing.T) { @@ -16,7 +17,7 @@ func TestPromptExistingConfigModeCLIMissingFileDefaultsToOverwrite(t *testing.T) if err != nil { t.Fatalf("promptExistingConfigModeCLI error: %v", err) } - if mode != existingConfigOverwrite { + if mode != installer.ExistingConfigOverwrite { t.Fatalf("expected overwrite mode, got %v", mode) } } @@ -30,7 +31,7 @@ func TestPromptExistingConfigModeCLIMissingFileRespectsCanceledContext(t *testin if !errors.Is(err, context.Canceled) { t.Fatalf("expected context canceled error, got %v", err) } - if mode != existingConfigCancel { + if mode != installer.ExistingConfigCancel { t.Fatalf("expected cancel mode, got %v", mode) } } @@ -40,21 +41,21 @@ func TestPromptExistingConfigModeCLIOptions(t *testing.T) { tests := []struct { name string input string - want existingConfigMode + want installer.ExistingConfigAction }{ - {name: "default keep continue", input: "\n", want: existingConfigKeepContinue}, - {name: "overwrite", input: "1\n", want: existingConfigOverwrite}, - {name: "edit", input: "2\n", want: existingConfigEdit}, - {name: "keep continue", input: "3\n", want: existingConfigKeepContinue}, - {name: "cancel", input: "0\n", want: existingConfigCancel}, - {name: "invalid then overwrite", input: "x\n1\n", want: existingConfigOverwrite}, + {name: "default keep continue", input: "\n", want: installer.ExistingConfigKeepContinue}, + {name: "overwrite", input: "1\n", want: installer.ExistingConfigOverwrite}, + {name: "edit", input: "2\n", want: installer.ExistingConfigEdit}, + {name: "keep continue", input: "3\n", want: installer.ExistingConfigKeepContinue}, + {name: "cancel", input: "0\n", want: installer.ExistingConfigCancel}, + {name: "invalid then overwrite", input: "x\n1\n", want: installer.ExistingConfigOverwrite}, } for _, tc := range tests { tc := tc t.Run(tc.name, func(t *testing.T) { reader := bufio.NewReader(strings.NewReader(tc.input)) - var mode existingConfigMode + var mode installer.ExistingConfigAction var err error captureStdout(t, func() { mode, err = promptExistingConfigModeCLI(context.Background(), reader, cfgFile) @@ -69,48 +70,6 @@ func TestPromptExistingConfigModeCLIOptions(t *testing.T) { } } -func TestResolveExistingConfigDecision(t *testing.T) { - cfgFile := createTempFile(t, "EXISTING=1\n") - - overwrite, err := resolveExistingConfigDecision(existingConfigOverwrite, cfgFile) - if err != nil { - t.Fatalf("overwrite decision error: %v", err) - } - if overwrite.SkipConfigWizard || overwrite.AbortInstall { - t.Fatalf("overwrite decision flags are invalid: %+v", overwrite) - } - if strings.TrimSpace(overwrite.BaseTemplate) == "" { - t.Fatalf("overwrite base template should not be empty") - } - - edit, err := resolveExistingConfigDecision(existingConfigEdit, cfgFile) - if err != nil { - t.Fatalf("edit decision error: %v", err) - } - if edit.SkipConfigWizard || edit.AbortInstall { - t.Fatalf("edit decision flags are invalid: %+v", edit) - } - if !strings.Contains(edit.BaseTemplate, "EXISTING=1") { - t.Fatalf("expected existing content, got %q", edit.BaseTemplate) - } - - keep, err := resolveExistingConfigDecision(existingConfigKeepContinue, cfgFile) - if err != nil { - t.Fatalf("keep decision error: %v", err) - } - if !keep.SkipConfigWizard || keep.AbortInstall { - t.Fatalf("keep decision flags are invalid: %+v", keep) - } - - cancel, err := resolveExistingConfigDecision(existingConfigCancel, cfgFile) - if err != nil { - t.Fatalf("cancel decision error: %v", err) - } - if cancel.SkipConfigWizard || !cancel.AbortInstall { - t.Fatalf("cancel decision flags are invalid: %+v", cancel) - } -} - func TestPrepareExistingConfigDecisionCLICancel(t *testing.T) { cfgFile := createTempFile(t, "EXISTING=1\n") reader := bufio.NewReader(strings.NewReader("0\n")) @@ -123,14 +82,6 @@ func TestPrepareExistingConfigDecisionCLICancel(t *testing.T) { } } -func TestResolveExistingConfigDecisionEditReadError(t *testing.T) { - cfgFile := filepath.Join(t.TempDir(), "missing.env") - _, err := resolveExistingConfigDecision(existingConfigEdit, cfgFile) - if err == nil { - t.Fatalf("expected read error for missing file") - } -} - func TestPromptExistingConfigModeCLIPropagatesReadError(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() @@ -152,14 +103,6 @@ func TestPromptExistingConfigModeCLINonRegularFile(t *testing.T) { } } -func TestResolveExistingConfigDecisionUnsupportedMode(t *testing.T) { - cfgFile := createTempFile(t, "EXISTING=1\n") - _, err := resolveExistingConfigDecision(existingConfigMode(99), cfgFile) - if err == nil { - t.Fatalf("expected unsupported mode error") - } -} - func TestPromptExistingConfigModeCLIStatError(t *testing.T) { pathWithNul := string([]byte{0}) _, err := promptExistingConfigModeCLI(context.Background(), bufio.NewReader(strings.NewReader("1\n")), pathWithNul) @@ -167,18 +110,3 @@ func TestPromptExistingConfigModeCLIStatError(t *testing.T) { t.Fatalf("expected stat error") } } - -func TestResolveExistingConfigDecisionEditExistingContentExact(t *testing.T) { - cfg := filepath.Join(t.TempDir(), "backup.env") - content := "KEY=VALUE\nANOTHER=1\n" - if err := os.WriteFile(cfg, []byte(content), 0o600); err != nil { - t.Fatalf("write config: %v", err) - } - decision, err := resolveExistingConfigDecision(existingConfigEdit, cfg) - if err != nil { - t.Fatalf("resolveExistingConfigDecision error: %v", err) - } - if decision.BaseTemplate != content { - t.Fatalf("expected exact content, got %q", decision.BaseTemplate) - } -} diff --git a/cmd/proxsave/install_test.go b/cmd/proxsave/install_test.go index ec3214a1..8ee4af7a 100644 --- a/cmd/proxsave/install_test.go +++ b/cmd/proxsave/install_test.go @@ -869,3 +869,41 @@ func parseWrittenEnvForTest(content string) map[string]string { } return values } + +// TestPrepareBaseTemplateEditBlankBaseStaysRaw pins the conditionality of the +// BaseTemplateOrDefault expansion, not merely its presence. Editing a blank +// backup.env must keep the base RAW: expanding it here would rewrite that file as +// the full embedded template instead of the minimal key set the wizard produces +// today, and would flip ApplyInstallData's editingExisting for that base. +// +// Without this test the gate is unpinned -- making the expansion unconditional +// leaves the whole suite, characterization goldens included, green. +func TestPrepareBaseTemplateEditBlankBaseStaysRaw(t *testing.T) { + for _, tc := range []struct { + name string + content string + }{ + {"zero byte", ""}, + {"whitespace only", " \n\t\n"}, + } { + t.Run(tc.name, func(t *testing.T) { + cfgFile := createTempFile(t, tc.content) + reader := bufio.NewReader(strings.NewReader("2\n")) + var tmpl string + var fromExisting bool + var err error + captureStdout(t, func() { + tmpl, _, fromExisting, err = prepareBaseTemplate(context.Background(), reader, cfgFile, nil) + }) + if err != nil { + t.Fatalf("prepareBaseTemplate error: %v", err) + } + if !fromExisting { + t.Fatal("edit must report fromExisting=true") + } + if tmpl != tc.content { + t.Fatalf("blank base must stay raw; got %d bytes (%q), want the file content back", len(tmpl), tmpl) + } + }) + } +} diff --git a/cmd/proxsave/install_tui.go b/cmd/proxsave/install_tui.go index 8b897799..d516c289 100644 --- a/cmd/proxsave/install_tui.go +++ b/cmd/proxsave/install_tui.go @@ -12,7 +12,6 @@ import ( "github.com/tis24dev/proxsave/internal/installer" "github.com/tis24dev/proxsave/internal/logging" "github.com/tis24dev/proxsave/internal/orchestrator" - "github.com/tis24dev/proxsave/internal/safefs" "github.com/tis24dev/proxsave/internal/ui/components" "github.com/tis24dev/proxsave/internal/ui/flows/agesetup" flowinstall "github.com/tis24dev/proxsave/internal/ui/flows/install" @@ -126,36 +125,32 @@ func runInstallTUI(ctx context.Context, configPath string, bootstrap *logging.Bo return mapUIDeath(err) } + decision, err := installer.ResolveExistingConfigDecision(existingAction, configPath) + if err != nil { + return err + } + var skipConfigWizard bool var wizardData *installer.InstallWizardData baseTemplate := "" - // Adopt the run time from the cron line this install is about to rewrite, now that - // the operator's answer is known. Nothing is written to backup.env here: on Edit - // the value is mirrored into baseTemplate below and ApplyInstallData rewrites the - // file at the end, so cancelling later leaves the host byte-identical. Keep - // existing has no wizard to carry it, so its write is deferred to the commit point. - schedulerSeed := deriveSchedulerTimeForExistingConfig(ctx, existingAction, configPath, bootstrap) - - switch existingAction { - case installer.ExistingConfigCancel: + switch { + case decision.AbortInstall: logging.DebugStepBootstrap(bootstrap, "install workflow (tui)", "user cancelled installation") return wrapInstallError(errInteractiveAborted) - case installer.ExistingConfigKeepContinue: + case decision.SkipConfigWizard: logging.DebugStepBootstrap(bootstrap, "install workflow (tui)", "using existing configuration and skipping wizard") skipConfigWizard = true - case installer.ExistingConfigEdit: + case decision.FromExistingFile: logging.DebugStepBootstrap(bootstrap, "install workflow (tui)", "editing existing configuration") - content, readErr := safefs.ReadFileUnderRoot(configPath) - if readErr != nil { - return fmt.Errorf("read existing configuration: %w", readErr) - } - baseTemplate = string(content) - if schedulerSeed.Time != "" { - // cronFieldDefault reads this to prefill "Run at"; without it the wizard - // would offer the 02:00 default instead of the host's real run time. - baseTemplate = setEnvValue(baseTemplate, "SCHEDULER_TIME", schedulerSeed.Time) - } + // Adopt the run time from the cron line this install is about to rewrite, now + // that the operator's answer is known. Nothing is written to backup.env here: + // ApplyInstallData rewrites the file at the end from this template, so + // cancelling later leaves the host byte-identical. cronFieldDefault reads the + // mirrored value to prefill "Run at"; without it the wizard would offer the + // 02:00 default instead of the host's real run time. Keep existing has no + // wizard to carry it, so its write stays deferred to the commit point. + baseTemplate = adoptCronRunTimeIntoBase(ctx, decision, configPath, bootstrap) default: logging.DebugStepBootstrap(bootstrap, "install workflow (tui)", "using embedded template") // Overwrite: use embedded template (handled as empty base) diff --git a/cmd/proxsave/schedule_helpers.go b/cmd/proxsave/schedule_helpers.go index 0467467b..575280dd 100644 --- a/cmd/proxsave/schedule_helpers.go +++ b/cmd/proxsave/schedule_helpers.go @@ -169,29 +169,35 @@ func schedulerTimeFromCronLines(lines []string) (string, bool) { return found, found != "" } -// deriveSchedulerTimeForExistingConfig is the read-only twin used by the TUI before -// the wizard runs: same gate, no write to backup.env. -func deriveSchedulerTimeForExistingConfig(ctx context.Context, action installer.ExistingConfigAction, configPath string, bootstrap *logging.BootstrapLogger) schedulerTimeSeed { - if !existingConfigAdoptsCronTime(action) { - return schedulerTimeSeed{} +// adoptCronRunTimeIntoBase is the ONE place both front-ends adopt the host's cron +// run time into the wizard's in-memory base. It returns the (possibly seeded) base +// and writes nothing to disk: on Edit the wizard rewrites the whole file at the end +// from this template, so an install cancelled halfway leaves the host byte-identical. +// +// The gate is decision.FromExistingFile, i.e. Edit ONLY. Cancel must leave the host +// untouched and Overwrite is about to replace the file, so an adoption note there +// would describe a value nobody will use. Keep existing has no wizard to carry the +// value, so its write stays deferred to the commit point in runInstall/runInstallTUI, +// which is also the single place its note is logged. +func adoptCronRunTimeIntoBase(ctx context.Context, decision installer.ExistingConfigDecision, configPath string, bootstrap *logging.BootstrapLogger) string { + if !decision.FromExistingFile { + return decision.BaseTemplate } seed := deriveSchedulerTimeFromCrontabFn(ctx, configPath) - if seed.Note != "" { + if seed.Note == "" { + return decision.BaseTemplate + } + seeded := installer.ApplySchedulerTimeSeed(decision.BaseTemplate, seed.Time) + // The adoption note promises "the daily run time does not change", so it may + // only be logged when the value actually reached the base: + // ApplySchedulerTimeSeed discards it on a blank base (see its guard), and a + // note the code then contradicts is worse than silence. The other variant + // carries no Time -- it warns that the cron entry could not be interpreted -- + // and is truthful whatever the base looks like. + if seed.Time == "" || seeded != decision.BaseTemplate { logBootstrapInfo(bootstrap, "%s", seed.Note) } - return seed -} - -// existingConfigAdoptsCronTime reports whether this answer commits to the existing -// backup.env. Cancel must leave the host untouched, and Overwrite is about to replace -// the file, so an adoption note there would describe a value nobody will use. -func existingConfigAdoptsCronTime(action installer.ExistingConfigAction) bool { - switch action { - case installer.ExistingConfigKeepContinue, installer.ExistingConfigEdit: - return true - default: - return false - } + return seeded } // hasProxsaveCronLine reports whether the crontab schedules proxsave at all (used diff --git a/cmd/proxsave/scheduler_time_seed_test.go b/cmd/proxsave/scheduler_time_seed_test.go index 8b934386..34301c93 100644 --- a/cmd/proxsave/scheduler_time_seed_test.go +++ b/cmd/proxsave/scheduler_time_seed_test.go @@ -12,6 +12,7 @@ import ( cronutil "github.com/tis24dev/proxsave/internal/cron" "github.com/tis24dev/proxsave/internal/installer" + "github.com/tis24dev/proxsave/internal/logging" ) // stubCrontabLines swaps the crontab read seam so the SCHEDULER_TIME seeding can be @@ -310,22 +311,30 @@ func TestPrepareBaseTemplateNeverWritesDuringTheInteractivePhase(t *testing.T) { } } -// TestDeriveSchedulerTimeForExistingConfig is the TUI twin of the CLI test above: -// same gate on which answers adopt the crontab time, and the same rule that the -// interactive phase must not write backup.env. The TUI carries the value into -// baseTemplate instead, and ApplyInstallData persists it only if the wizard finishes. -func TestDeriveSchedulerTimeForExistingConfig(t *testing.T) { +// TestAdoptCronRunTimeIntoBase pins the ONE adoption helper both front-ends now +// share: which answers adopt the crontab time into the wizard's in-memory base, +// and the rule that the interactive phase must not write backup.env. The adopted +// value is carried in the returned base, and ApplyInstallData persists it only if +// the wizard finishes. +// +// S2: the gate is decision.FromExistingFile, i.e. EDIT ONLY - the CLI's behavior, +// which wins over the TUI's former Keep-OR-Edit gate. "keep existing" therefore +// adopts NOTHING here; that is the signed-off unification, not a weakened test. +// Keep still gets its SCHEDULER_TIME, but at the commit point via +// seedSchedulerTimeFromCrontabFn (install.go / install_tui.go), which is unchanged +// and is now the single place the Keep-path note is logged instead of twice. +func TestAdoptCronRunTimeIntoBase(t *testing.T) { const preThirty = "SCHEDULER_MODE=cron\n" tests := []struct { - name string - action installer.ExistingConfigAction - wantTime string + name string + action installer.ExistingConfigAction + wantAdopts bool }{ {name: "cancel adopts nothing", action: installer.ExistingConfigCancel}, {name: "overwrite adopts nothing", action: installer.ExistingConfigOverwrite}, - {name: "edit adopts", action: installer.ExistingConfigEdit, wantTime: "21:00"}, - {name: "keep existing adopts", action: installer.ExistingConfigKeepContinue, wantTime: "21:00"}, + {name: "edit adopts", action: installer.ExistingConfigEdit, wantAdopts: true}, + {name: "keep existing adopts nothing (S2)", action: installer.ExistingConfigKeepContinue}, } for _, tt := range tests { @@ -336,12 +345,35 @@ func TestDeriveSchedulerTimeForExistingConfig(t *testing.T) { } stubCrontabLines(t, []string{"0 21 * * * /usr/local/bin/proxsave --backup"}, nil) - seed := deriveSchedulerTimeForExistingConfig(context.Background(), tt.action, cfg, nil) - if seed.Time != tt.wantTime { - t.Fatalf("seed.Time = %q, want %q", seed.Time, tt.wantTime) + decision, err := installer.ResolveExistingConfigDecision(tt.action, cfg) + if err != nil { + t.Fatalf("ResolveExistingConfigDecision error: %v", err) + } + + // The resolver hands a RAW "" base to every answer except Edit, and + // ApplySchedulerTimeSeed is a no-op on "" -- so asserting on the returned + // base alone would be VACUOUS for three of the four rows: it reads false + // whether the gate is Edit-only or the TUI's former Keep-OR-Edit. Count + // the derive instead: it is the observable the gate actually controls, and + // it fails if the gate is removed. + derives := 0 + origDerive := deriveSchedulerTimeFromCrontabFn + deriveSchedulerTimeFromCrontabFn = func(ctx context.Context, path string) schedulerTimeSeed { + derives++ + return origDerive(ctx, path) + } + t.Cleanup(func() { deriveSchedulerTimeFromCrontabFn = origDerive }) + + base := adoptCronRunTimeIntoBase(context.Background(), decision, cfg, nil) + if got := strings.Contains(base, "SCHEDULER_TIME=21:00"); got != tt.wantAdopts { + t.Fatalf("adopted = %v, want %v (base=%q)", got, tt.wantAdopts, base) } - if tt.wantTime == "" && seed.Note != "" { - t.Errorf("no adoption note expected for this answer, got %q", seed.Note) + wantDerives := 0 + if tt.wantAdopts { + wantDerives = 1 + } + if derives != wantDerives { + t.Fatalf("crontab derive calls = %d, want %d: only Edit may consult the crontab (S2)", derives, wantDerives) } data, err := os.ReadFile(cfg) @@ -418,3 +450,46 @@ func TestApplyConfigUpgradeKeepsExplicitSchedulerTime(t *testing.T) { t.Errorf("no adoption note expected when the operator set the time: %v", result.Warnings) } } + +// TestAdoptCronRunTimeIntoBaseNoteMatchesReality pins that the adoption note is +// only emitted when the value actually reached the base. The note promises "the +// daily run time does not change", but ApplySchedulerTimeSeed discards the seed on +// a blank base -- so logging unconditionally told the operator their 21:00 was kept +// while the wizard went on to offer the 02:00 default. A note the code contradicts +// is worse than silence. +func TestAdoptCronRunTimeIntoBaseNoteMatchesReality(t *testing.T) { + for _, tt := range []struct { + name string + content string + wantSeeded bool + }{ + {name: "blank base discards the seed, so no note", content: "", wantSeeded: false}, + {name: "real base keeps the seed and the note", content: "SCHEDULER_MODE=cron\n", wantSeeded: true}, + } { + t.Run(tt.name, func(t *testing.T) { + cfg := filepath.Join(t.TempDir(), "backup.env") + if err := os.WriteFile(cfg, []byte(tt.content), 0o600); err != nil { + t.Fatalf("seed config: %v", err) + } + stubCrontabLines(t, []string{"0 21 * * * /usr/local/bin/proxsave --backup"}, nil) + + decision, err := installer.ResolveExistingConfigDecision(installer.ExistingConfigEdit, cfg) + if err != nil { + t.Fatalf("ResolveExistingConfigDecision error: %v", err) + } + bootstrap := logging.NewBootstrapLogger() + base := adoptCronRunTimeIntoBase(context.Background(), decision, cfg, bootstrap) + + if got := strings.Contains(base, "SCHEDULER_TIME=21:00"); got != tt.wantSeeded { + t.Fatalf("seeded = %v, want %v (base=%q)", got, tt.wantSeeded, base) + } + wantEntries := 0 + if tt.wantSeeded { + wantEntries = 1 + } + if got := bootstrap.EntryCount(); got != wantEntries { + t.Fatalf("bootstrap entries = %d, want %d: the adoption note must not outlive the value it describes", got, wantEntries) + } + }) + } +} diff --git a/internal/installer/existing_config.go b/internal/installer/existing_config.go new file mode 100644 index 00000000..f0e193ca --- /dev/null +++ b/internal/installer/existing_config.go @@ -0,0 +1,148 @@ +package installer + +import ( + "fmt" + "os" + "strings" + + "github.com/tis24dev/proxsave/internal/config" + "github.com/tis24dev/proxsave/internal/safefs" +) + +// ExistingConfigAction represents how to handle an already-present +// configuration file. +type ExistingConfigAction int + +const ( + ExistingConfigOverwrite ExistingConfigAction = iota // Start from embedded template (overwrite) + ExistingConfigEdit // Keep existing file as base and edit + ExistingConfigKeepContinue // Leave file untouched and continue installation + ExistingConfigCancel // Abort installation +) + +// ExistingConfigDecision is the engine-side outcome of the existing-config +// question, shared by the CLI prompt (cmd/proxsave/install_existing_config.go) +// and the Charm screen (internal/ui/flows/install.ResolveExistingConfig). +// +// BaseTemplate is the RAW base: "" means "no base, use the embedded default". +// That emptiness is LOAD-BEARING and must not be expanded before it reaches +// ApplyInstallData, which derives editingExisting from +// strings.TrimSpace(baseTemplate) != "": handing it an expanded default on +// Overwrite would silently flip editingExisting to true and change which keys +// are preserved. A front-end that drives its OWN prompts off the base (the CLI +// wizard) calls BaseTemplateOrDefault for that purpose only. +type ExistingConfigDecision struct { + // BaseTemplate is the raw wizard base; "" means the embedded default. + BaseTemplate string + // SkipConfigWizard is set by KeepContinue: leave backup.env untouched. + SkipConfigWizard bool + // AbortInstall is set by Cancel: the caller must abort without changes. + AbortInstall bool + // FromExistingFile is true ONLY for Edit, i.e. the wizard starts from the + // operator's current backup.env. Fresh installs and Overwrite start from the + // embedded template, so defaults (e.g. the scheduler engine) may be the + // recommended new values rather than the stored ones. It is also the single + // gate for adopting the crontab run time (see ApplySchedulerTimeSeed). + FromExistingFile bool +} + +// ExistingConfigPresent is the stat pre-check both front-ends run before asking +// anything. +// +// (false, nil) = no file: a fresh install, the caller proceeds as Overwrite +// without showing any prompt. (true, nil) = a regular file the operator must +// decide about. (false, err) = the path is unusable. Callers keep their own +// context handling (the CLI still checks ctx.Err() on the no-file path before +// returning Overwrite). +func ExistingConfigPresent(configPath string) (bool, error) { + info, err := os.Stat(configPath) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, fmt.Errorf("failed to access configuration file: %w", err) + } + if !info.Mode().IsRegular() { + return false, fmt.Errorf("configuration file path is not a regular file: %s", configPath) + } + return true, nil +} + +// ResolveExistingConfigDecision turns the operator's answer into the engine-side +// decision. Overwrite yields the RAW empty base (see ExistingConfigDecision) and +// NOT config.DefaultEnvTemplate(); Edit reads the current file and sets +// FromExistingFile; KeepContinue skips the wizard; Cancel aborts. An unknown +// action is a programming error. +func ResolveExistingConfigDecision(action ExistingConfigAction, configPath string) (ExistingConfigDecision, error) { + switch action { + case ExistingConfigOverwrite: + return ExistingConfigDecision{ + BaseTemplate: "", + SkipConfigWizard: false, + AbortInstall: false, + }, nil + case ExistingConfigEdit: + content, err := safefs.ReadFileUnderRoot(configPath) + if err != nil { + return ExistingConfigDecision{}, fmt.Errorf("read existing configuration: %w", err) + } + return ExistingConfigDecision{ + BaseTemplate: string(content), + SkipConfigWizard: false, + AbortInstall: false, + FromExistingFile: true, + }, nil + case ExistingConfigKeepContinue: + return ExistingConfigDecision{ + BaseTemplate: "", + SkipConfigWizard: true, + AbortInstall: false, + }, nil + case ExistingConfigCancel: + return ExistingConfigDecision{ + BaseTemplate: "", + SkipConfigWizard: false, + AbortInstall: true, + }, nil + default: + return ExistingConfigDecision{}, fmt.Errorf("unsupported existing configuration action: %d", action) + } +} + +// BaseTemplateOrDefault expands an empty RAW base into the embedded template. +// It exists for front-ends that compute their own prompt defaults from the base +// (the CLI wizard, whose Overwrite path used to receive an already-expanded +// template). ApplyInstallData performs the same substitution internally, so what +// is handed to IT must stay raw. +// +// It must be called ONLY when !FromExistingFile. On the Edit path a blank or +// whitespace-only backup.env is a real (if odd) operator state: expanding it here +// would rewrite that file as the FULL embedded template instead of the minimal +// mutated key set it produces today, and would flip ApplyInstallData's +// editingExisting for that base, changing which keys are treated as pre-existing. +func BaseTemplateOrDefault(base string) string { + if strings.TrimSpace(base) == "" { + return config.DefaultEnvTemplate() + } + return base +} + +// ApplySchedulerTimeSeed mirrors a run time adopted from the host's existing +// proxsave cron line into the wizard's in-memory base, so the "Run at" prompt +// offers the host's real time instead of the 02:00 template default. It writes +// nothing to disk. +// +// The base=="" guard is the CLI's existing guard and is deliberately an +// EXACT-empty test, not strings.TrimSpace. Without it, seeding a "" base +// produces "\nSCHEDULER_TIME=HH:MM", which flips ApplyInstallData's +// editingExisting to true, defeats its blank->embedded-default substitution and +// writes a gutted config. KNOWN RESIDUE: a whitespace-only base is != "", so it +// is still mirrored into and still gutted; fixing that requires a TrimSpace test +// that would also change the CLI (a whitespace-only backup.env + Edit would stop +// adopting the crontab time), so it is out of scope here. +func ApplySchedulerTimeSeed(base, hhmm string) string { + if hhmm == "" || base == "" { + return base + } + return setEnvValue(base, "SCHEDULER_TIME", hhmm) +} diff --git a/internal/installer/existing_config_test.go b/internal/installer/existing_config_test.go new file mode 100644 index 00000000..f0a304fb --- /dev/null +++ b/internal/installer/existing_config_test.go @@ -0,0 +1,156 @@ +package installer + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func writeExistingConfig(t *testing.T, content string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "backup.env") + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatalf("write config: %v", err) + } + return path +} + +func TestResolveExistingConfigDecision(t *testing.T) { + cfgFile := writeExistingConfig(t, "EXISTING=1\n") + + overwrite, err := ResolveExistingConfigDecision(ExistingConfigOverwrite, cfgFile) + if err != nil { + t.Fatalf("overwrite decision error: %v", err) + } + if overwrite.SkipConfigWizard || overwrite.AbortInstall || overwrite.FromExistingFile { + t.Fatalf("overwrite decision flags are invalid: %+v", overwrite) + } + // S1: the decision carries the RAW base. "" is load-bearing - ApplyInstallData + // derives editingExisting from strings.TrimSpace(baseTemplate) != "", so an + // expanded default here would flip it to true and change which keys are + // preserved. Front-ends that drive their own prompts off the base expand it + // themselves via BaseTemplateOrDefault. + if overwrite.BaseTemplate != "" { + t.Fatalf("overwrite base template must stay raw/empty, got %q", overwrite.BaseTemplate) + } + if BaseTemplateOrDefault(overwrite.BaseTemplate) == "" { + t.Fatalf("BaseTemplateOrDefault must expand the empty overwrite base") + } + + edit, err := ResolveExistingConfigDecision(ExistingConfigEdit, cfgFile) + if err != nil { + t.Fatalf("edit decision error: %v", err) + } + if edit.SkipConfigWizard || edit.AbortInstall || !edit.FromExistingFile { + t.Fatalf("edit decision flags are invalid: %+v", edit) + } + if !strings.Contains(edit.BaseTemplate, "EXISTING=1") { + t.Fatalf("expected existing content, got %q", edit.BaseTemplate) + } + + keep, err := ResolveExistingConfigDecision(ExistingConfigKeepContinue, cfgFile) + if err != nil { + t.Fatalf("keep decision error: %v", err) + } + if !keep.SkipConfigWizard || keep.AbortInstall || keep.FromExistingFile { + t.Fatalf("keep decision flags are invalid: %+v", keep) + } + + cancel, err := ResolveExistingConfigDecision(ExistingConfigCancel, cfgFile) + if err != nil { + t.Fatalf("cancel decision error: %v", err) + } + if cancel.SkipConfigWizard || !cancel.AbortInstall || cancel.FromExistingFile { + t.Fatalf("cancel decision flags are invalid: %+v", cancel) + } +} + +func TestResolveExistingConfigDecisionEditReadError(t *testing.T) { + cfgFile := filepath.Join(t.TempDir(), "missing.env") + if _, err := ResolveExistingConfigDecision(ExistingConfigEdit, cfgFile); err == nil { + t.Fatalf("expected read error for missing file") + } +} + +func TestResolveExistingConfigDecisionUnsupportedAction(t *testing.T) { + cfgFile := writeExistingConfig(t, "EXISTING=1\n") + if _, err := ResolveExistingConfigDecision(ExistingConfigAction(99), cfgFile); err == nil { + t.Fatalf("expected unsupported action error") + } +} + +func TestResolveExistingConfigDecisionEditExistingContentExact(t *testing.T) { + content := "KEY=VALUE\nANOTHER=1\n" + cfg := writeExistingConfig(t, content) + decision, err := ResolveExistingConfigDecision(ExistingConfigEdit, cfg) + if err != nil { + t.Fatalf("ResolveExistingConfigDecision error: %v", err) + } + if decision.BaseTemplate != content { + t.Fatalf("expected exact content, got %q", decision.BaseTemplate) + } +} + +func TestExistingConfigPresent(t *testing.T) { + cfg := writeExistingConfig(t, "EXISTING=1\n") + present, err := ExistingConfigPresent(cfg) + if err != nil { + t.Fatalf("ExistingConfigPresent error: %v", err) + } + if !present { + t.Fatalf("expected a regular file to be reported present") + } + + missing := filepath.Join(t.TempDir(), "missing.env") + present, err = ExistingConfigPresent(missing) + if err != nil { + t.Fatalf("missing file must not be an error, got %v", err) + } + if present { + t.Fatalf("expected a missing file to be reported absent") + } + + present, err = ExistingConfigPresent(t.TempDir()) + if err == nil { + t.Fatalf("expected error for a non-regular file") + } + if present { + t.Fatalf("a non-regular file must not be reported present") + } + if !strings.Contains(err.Error(), "not a regular file") { + t.Fatalf("unexpected error message: %v", err) + } +} + +// TestApplySchedulerTimeSeedEmptyBase pins S3: the mirror keeps the CLI's +// exact-empty guard. Without it a "" base becomes "\nSCHEDULER_TIME=HH:MM", +// which flips ApplyInstallData's editingExisting to true, defeats its +// blank->embedded-default substitution and writes a gutted config. +func TestApplySchedulerTimeSeedEmptyBase(t *testing.T) { + if got := ApplySchedulerTimeSeed("", "21:00"); got != "" { + t.Fatalf("empty base must stay empty, got %q", got) + } + if got := ApplySchedulerTimeSeed("SCHEDULER_MODE=cron\n", ""); got != "SCHEDULER_MODE=cron\n" { + t.Fatalf("empty time must leave the base untouched, got %q", got) + } +} + +func TestApplySchedulerTimeSeedMirrorsTime(t *testing.T) { + got := ApplySchedulerTimeSeed("SCHEDULER_MODE=cron\n", "21:00") + if !strings.Contains(got, "SCHEDULER_TIME=21:00") { + t.Fatalf("expected SCHEDULER_TIME=21:00 in %q", got) + } + if !strings.Contains(got, "SCHEDULER_MODE=cron") { + t.Fatalf("expected the existing base to survive, got %q", got) + } +} + +func TestBaseTemplateOrDefault(t *testing.T) { + if BaseTemplateOrDefault("") == "" { + t.Fatalf("empty base must expand to the embedded template") + } + if got := BaseTemplateOrDefault("KEY=VALUE\n"); got != "KEY=VALUE\n" { + t.Fatalf("a non-empty base must pass through unchanged, got %q", got) + } +} diff --git a/internal/installer/ui_support.go b/internal/installer/ui_support.go index f2927952..3e20b31c 100644 --- a/internal/installer/ui_support.go +++ b/internal/installer/ui_support.go @@ -15,17 +15,6 @@ import ( // ErrInstallCancelled is returned when the user aborts the install wizard. var ErrInstallCancelled = errors.New("installation aborted by user") -// ExistingConfigAction represents how to handle an already-present -// configuration file. -type ExistingConfigAction int - -const ( - ExistingConfigOverwrite ExistingConfigAction = iota // Start from embedded template (overwrite) - ExistingConfigEdit // Keep existing file as base and edit - ExistingConfigKeepContinue // Leave file untouched and continue installation - ExistingConfigCancel // Abort installation -) - // PostInstallAuditResult reports the outcome of the optional post-install // audit step. type PostInstallAuditResult struct { diff --git a/internal/ui/flows/install/install.go b/internal/ui/flows/install/install.go index 0236e932..cc3c1c0c 100644 --- a/internal/ui/flows/install/install.go +++ b/internal/ui/flows/install/install.go @@ -9,7 +9,6 @@ package install import ( "context" "fmt" - "os" "strings" "github.com/tis24dev/proxsave/internal/config" @@ -32,15 +31,12 @@ func mapCancel(err error) error { // file. When no file exists it returns Overwrite without any screen (same // contract as the tview CheckExistingConfig and the CLI prompt). func ResolveExistingConfig(ctx context.Context, session *shell.Session, configPath string) (installer.ExistingConfigAction, error) { - info, err := os.Stat(configPath) + exists, err := installer.ExistingConfigPresent(configPath) if err != nil { - if os.IsNotExist(err) { - return installer.ExistingConfigOverwrite, nil - } - return installer.ExistingConfigCancel, fmt.Errorf("failed to access configuration file: %w", err) + return installer.ExistingConfigCancel, err } - if !info.Mode().IsRegular() { - return installer.ExistingConfigCancel, fmt.Errorf("configuration file path is not a regular file: %s", configPath) + if !exists { + return installer.ExistingConfigOverwrite, nil } items := []components.SelectorItem[installer.ExistingConfigAction]{ From d717aa7eea05f78218f1c605c0ed11de8488aeeb Mon Sep 17 00:00:00 2001 From: tis24dev Date: Mon, 3 Aug 2026 01:07:51 +0200 Subject: [PATCH 08/50] refactor(install): CLI wizard collects InstallWizardData and applies it via the engine installer.ApplyInstallData had exactly one caller, the Charm front-end. The CLI re-implemented the same key-by-key logic inline -- secondary storage, cloud, firewall, notifications, encryption, scheduler, healthchecks -- and said so in its own comments ("mirroring the TUI's ApplyInstallData", "parity with the TUI's ApplyInstallData"). Those two implementations are now one: the CLI prompts collect an *installer.InstallWizardData and the engine writes the file. The prompts are untouched. Five configure* helpers become prompt-only (no template in, no template out) with every print string, re-prompt loop and sanitizeEnvValue placement copied verbatim, so the five characterization transcripts still match byte for byte. Byte-identity was previously argued; it is now demonstrated. A differential harness outside the repo drove the real runConfigWizardCLI and the real ApplyInstallData over the same inputs and compared the written bytes across 26 scenarios: the three golden .env files reproduce exactly from both sides, and so do cases no golden covers -- minimal bases where 14 keys are appended (which is what exercises append ORDER, the most fragile part of this change), runtime-derived-key removal, export form, no trailing newline, daemon+self keeping its ping URLs, self->off clearing them, and a legacy EMAIL_FALLBACK_PMF base. Two bug-compatibility mechanisms are load-bearing and must be removed together with the signed-off blank-base alignment, never independently: - wizardBlankEditBaseMarker. ApplyInstallData substitutes the whole embedded template when the base is blank, so an Edit of an empty backup.env would have grown from ~290 bytes to ~25 KB. The marker keeps the base non-blank across the call and is stripped afterwards. It also covers a base that is non-empty on disk but consists solely of runtime-derived keys, which the engine strips to nothing -- that case was writing one extra leading blank line. - wizardBlankBaseStandIn. The scheduler/healthcheck/run-at prompt defaults used to be read off the running template, which the secondary-storage step had already made non-blank. Feeding the raw blank base instead flips the transcript to [daemon]/[centralized] and consumes an extra scripted answer. One intentional delta: a cloud remote that sanitizeEnvValue empties is re-prompted instead of written as CLOUD_ENABLED=true with an empty CLOUD_REMOTE. The default is sanitized too -- a stored value made of control characters was offered as acceptable and could then never satisfy the check. This does not preserve the old outcome on a finite piped stream, where the extra prompt can now exhaust the answers; the comment says so rather than claiming otherwise. The raw/expanded base split is pinned by its own test because no golden can catch it: handing the engine the expanded base is byte-identical on every golden scenario, so a green suite is not evidence that the split is right. Out of scope, left bug-compatible and now tripwired so the follow-ups fail loudly: EMAIL_FALLBACK_SENDMAIL still forced true via a non-nil pointer, BACKUP_FIREWALL_RULES still always written, and the fresh-install cloud prompts still prefill the template placeholders. --- cmd/proxsave/config_helpers.go | 4 + cmd/proxsave/install.go | 466 ++++++++----- cmd/proxsave/install_test.go | 797 ++++++++++++++++++----- cmd/proxsave/scheduler_time_seed_test.go | 8 +- 4 files changed, 940 insertions(+), 335 deletions(-) diff --git a/cmd/proxsave/config_helpers.go b/cmd/proxsave/config_helpers.go index 0eb7d92c..b9956505 100644 --- a/cmd/proxsave/config_helpers.go +++ b/cmd/proxsave/config_helpers.go @@ -45,6 +45,10 @@ func ensureConfigExists(path string, logger configStatusLogger) error { return fmt.Errorf("configuration file is required to continue") } +// setEnvValue is TEST-ONLY since the install wizard started routing its writes +// through installer.ApplyInstallData: cmd/proxsave/install_characterization_test.go +// and cmd/proxsave/helpers_test.go build fixture templates with it. Production code +// must use installer.SetEnvValueInTemplate / the install engine instead. func setEnvValue(template, key, value string) string { return utils.SetEnvValue(template, key, value) } diff --git a/cmd/proxsave/install.go b/cmd/proxsave/install.go index 5aa408ba..a0b13365 100644 --- a/cmd/proxsave/install.go +++ b/cmd/proxsave/install.go @@ -489,7 +489,7 @@ func runConfigWizardCLI(ctx context.Context, reader *bufio.Reader, configPath, t defer func() { done(err) }() logging.DebugStepBootstrap(bootstrap, "install config wizard (cli)", "preparing base template") - template, skipConfigWizard, fromExisting, err := prepareBaseTemplate(ctx, reader, configPath, bootstrap) + base, skipConfigWizard, fromExisting, err := prepareBaseTemplate(ctx, reader, configPath, bootstrap) if err != nil { return installConfigResult{}, wrapInstallError(err) } @@ -498,101 +498,193 @@ func runConfigWizardCLI(ctx context.Context, reader *bufio.Reader, configPath, t return installConfigResult{SkipConfigWizard: true}, nil } - logging.DebugStepBootstrap(bootstrap, "install config wizard (cli)", "configuring secondary storage") - if template, err = configureSecondaryStorage(ctx, reader, template); err != nil { + data, err := collectInstallWizardDataCLI(ctx, reader, base.Prompt, fromExisting, bootstrap) + if err != nil { return installConfigResult{}, wrapInstallError(err) } + + // installConfigResult is a pure projection of the payload: the two can no + // longer disagree. HealthcheckMode in particular gates runHealthcheckSelfParamsCLI + // in runInstall. + result = installConfigResult{ + EnableEncryption: data.EnableEncryption, + CronSchedule: cronutil.TimeToSchedule(data.CronTime), + SchedulerMode: data.SchedulerMode, + HealthcheckMode: data.HealthcheckMode, + } + + if bootstrap != nil { + bootstrap.Info("Scheduler: %s, run at %s", data.SchedulerMode, data.CronTime) + } + + logging.DebugStepBootstrap(bootstrap, "install config wizard (cli)", "writing configuration") + // The RAW base goes to the engine: it derives editingExisting from it and + // substitutes the embedded default when it is blank. NOT wrapped in + // wrapInstallError - a rejected payload is a failure, not a user abort - but the + // prompt loops (promptSecondaryStorage, promptCloudStorage) already enforce + // everything ApplyInstallData validates, so this edge is unreachable. + template, err := applyInstallDataCLI(base, fromExisting, data) + if err != nil { + return installConfigResult{}, err + } + if err := installer.WriteConfigFileAtomic(configPath, tmpConfigPath, template); err != nil { + return installConfigResult{}, err + } + + if bootstrap != nil { + bootstrap.Info("✓ Configuration saved at %s", configPath) + } + + return result, nil +} + +// wizardBlankEditBaseMarker keeps an Edit of a blank (empty or whitespace-only) +// backup.env writing the minimal key set it has always written. +// installer.ApplyInstallData substitutes the whole embedded template whenever +// strings.TrimSpace(base) == "", so handing it the raw blank base would rewrite the +// operator's file as ~25 KB of defaults. Aligning that with the Charm front-end is a +// separate, signed-off behavior change, so F1 stays bug-compatible by prefixing a +// comment line the engine cannot touch and stripping it back off. +// +// Prefixing is safe because the marker only flips editingExisting, which is +// UNOBSERVABLE for a CLI payload: the blank base carries no BOT_TELEGRAM_TYPE (so the +// key is seeded either way), the collector always supplies a non-empty +// EmailDeliveryMethod, and EmailFallbackSendmail is always non-nil. Pinned by +// TestRunConfigWizardCLIBlankEditKeepsMinimalKeySet. +const wizardBlankEditBaseMarker = "# proxsave install wizard: blank existing configuration\n" + +func applyInstallDataCLI(base installWizardBase, fromExisting bool, data *installer.InstallWizardData) (string, error) { + // The base counts as blank not only when it is empty to begin with, but also when + // RemoveRuntimeDerivedEnvKeys empties it -- a backup.env consisting solely of + // BASE_DIR/CRON_* is stripped to nothing inside ApplyInstallData, and the appended + // keys then land after a leftover newline. Testing only TrimSpace(base.Raw) left + // that case writing one extra leading blank line. + if !fromExisting || strings.TrimSpace(config.RemoveRuntimeDerivedEnvKeys(base.Raw)) != "" { + return installer.ApplyInstallData(base.Raw, data) + } + template, err := installer.ApplyInstallData(wizardBlankEditBaseMarker+base.Raw, data) + if err != nil { + return "", err + } + return strings.TrimPrefix(template, wizardBlankEditBaseMarker), nil +} + +// wizardBlankBaseStandIn stands in for a blank Edit base when the CLI derives its +// PROMPT defaults (it never reaches backup.env). It is load-bearing, not cosmetic: +// the defaults used to be read off the RUNNING template, which the secondary-storage +// step had already made non-blank, so the empty-base branch of schedulerEngineDefault +// / healthcheckModeDefault / cronTimeDefault was dead at runtime. Feeding those +// helpers a genuinely blank base instead would flip the prompts to [daemon] and +// [centralized] and consume an extra scripted answer. Pinned by +// TestCollectInstallWizardDataCLIBlankEditKeepsStoredDefaults. +const wizardBlankBaseStandIn = "# (blank existing configuration)" + +// collectInstallWizardDataCLI asks every wizard question and returns the payload for +// installer.ApplyInstallData. It NEVER sees the raw base: promptBase is the expanded +// template the prompt defaults are read from, and only runConfigWizardCLI holds both. +// +// The prefill is derived ONCE instead of once per step. That is equivalent because +// every step's write-set is disjoint from every later step's read-set (secondary +// writes SECONDARY_*, cloud CLOUD_*, firewall BACKUP_FIREWALL_RULES, notifications +// TELEGRAM_*/BOT_TELEGRAM_TYPE/EMAIL_*, encryption ENCRYPT_ARCHIVE; the +// SCHEDULER_*/HEALTHCHECK_* keys the last three defaults read are never written +// before them). +func collectInstallWizardDataCLI(ctx context.Context, reader *bufio.Reader, promptBase string, fromExisting bool, bootstrap *logging.BootstrapLogger) (*installer.InstallWizardData, error) { + prefillBase := promptBase + if strings.TrimSpace(prefillBase) == "" { + prefillBase = wizardBlankBaseStandIn + } + prefill := installer.DeriveInstallWizardPrefill(prefillBase) + + data := &installer.InstallWizardData{} + + logging.DebugStepBootstrap(bootstrap, "install config wizard (cli)", "configuring secondary storage") + secondaryEnabled, secondaryPath, secondaryLogPath, err := promptSecondaryStorage(ctx, reader, prefill) + if err != nil { + return nil, err + } + data.EnableSecondaryStorage = secondaryEnabled + data.SecondaryPath = secondaryPath + data.SecondaryLogPath = secondaryLogPath + logging.DebugStepBootstrap(bootstrap, "install config wizard (cli)", "configuring cloud storage") - if template, err = configureCloudStorage(ctx, reader, template); err != nil { - return installConfigResult{}, wrapInstallError(err) + cloudEnabled, cloudRemote, cloudLogRemote, err := promptCloudStorage(ctx, reader, prefill) + if err != nil { + return nil, err } + data.EnableCloudStorage = cloudEnabled + data.RcloneBackupRemote = cloudRemote + data.RcloneLogRemote = cloudLogRemote + logging.DebugStepBootstrap(bootstrap, "install config wizard (cli)", "configuring firewall rules") - if template, err = configureFirewallRules(ctx, reader, template); err != nil { - return installConfigResult{}, wrapInstallError(err) + firewallEnabled, err := promptFirewallRules(ctx, reader, prefill) + if err != nil { + return nil, err } + // ALWAYS non-nil: the CLI has always written BACKUP_FIREWALL_RULES on every run, + // so the engine's nil "keep the stored value" branch must stay unreachable here. + data.BackupFirewallRules = &firewallEnabled + logging.DebugStepBootstrap(bootstrap, "install config wizard (cli)", "configuring notifications") - if template, err = configureNotifications(ctx, reader, template); err != nil { - return installConfigResult{}, wrapInstallError(err) + telegramEnabled, emailEnabled, emailMethod, err := promptNotifications(ctx, reader, prefill) + if err != nil { + return nil, err + } + switch { + case telegramEnabled && emailEnabled: + data.NotificationMode = "both" + case telegramEnabled: + data.NotificationMode = "telegram" + case emailEnabled: + data.NotificationMode = "email" + default: + data.NotificationMode = "none" + } + if emailEnabled { + data.EmailDeliveryMethod = emailMethod + // ALWAYS non-nil true, matching what the CLI has always written and what the + // Charm front-end sends. The engine's 3-branch preserve logic stays dead for + // both; switching to preserve semantics is a separate behavior change. + fallbackSendmail := true + data.EmailFallbackSendmail = &fallbackSendmail } logging.DebugStepBootstrap(bootstrap, "install config wizard (cli)", "configuring encryption") - result.EnableEncryption, err = configureEncryption(ctx, reader, &template) + data.EnableEncryption, err = promptEncryption(ctx, reader, prefill) if err != nil { - return installConfigResult{}, wrapInstallError(err) + return nil, err } logging.DebugStepBootstrap(bootstrap, "install config wizard (cli)", "configuring scheduler engine") - engine, err := configureSchedulerEngine(ctx, reader, schedulerEngineDefault(fromExisting, template)) + engine, err := configureSchedulerEngine(ctx, reader, schedulerEngineDefault(fromExisting, prefillBase)) if err != nil { - return installConfigResult{}, wrapInstallError(err) + return nil, err } - result.SchedulerMode = engine + data.SchedulerMode = engine // Healthchecks require the daemon (the sole pinger); with cron the mode is // forced off and no prompt is shown, mirroring the TUI's Active gate. - hcMode := "off" + data.HealthcheckMode = "off" if engine == "daemon" { logging.DebugStepBootstrap(bootstrap, "install config wizard (cli)", "configuring healthcheck mode") - hcMode, err = configureHealthcheckMode(ctx, reader, healthcheckModeDefault(fromExisting, template)) + hcMode, err := configureHealthcheckMode(ctx, reader, healthcheckModeDefault(fromExisting, prefillBase)) if err != nil { - return installConfigResult{}, wrapInstallError(err) + return nil, err } + data.HealthcheckMode = hcMode } - result.HealthcheckMode = hcMode logging.DebugStepBootstrap(bootstrap, "install config wizard (cli)", "configuring run-at time") - cronTime, err := configureCronTimeFunc(ctx, reader, cronTimeDefault(fromExisting, template)) + // configureCronTimeFunc, not configureCronTime: the package-level seam is stubbed + // by cmd/proxsave/install_test.go. It always returns a normalized HH:MM, which is + // what keeps ApplyInstallData's "skip SCHEDULER_TIME when blank" branch unreachable. + cronTime, err := configureCronTimeFunc(ctx, reader, cronTimeDefault(fromExisting, prefillBase)) if err != nil { - return installConfigResult{}, wrapInstallError(err) + return nil, err } - result.CronSchedule = cronutil.TimeToSchedule(cronTime) + data.CronTime = cronTime - if bootstrap != nil { - bootstrap.Info("Scheduler: %s, run at %s", engine, cronTime) - } - - logging.DebugStepBootstrap(bootstrap, "install config wizard (cli)", "writing configuration") - template = config.RemoveRuntimeDerivedEnvKeys(template) - template = setEnvValue(template, "SCHEDULER_MODE", engine) - template = setEnvValue(template, "SCHEDULER_TIME", cronTime) - // Write HEALTHCHECK_ENABLED + HEALTHCHECK_MODE from the explicit choice (cron - // and "off" disable it), replacing the old implicit daemon->enabled rule. The - // self-mode ping URLs are collected by runHealthcheckSelfParamsCLI after this - // write, before the healthcheck bootstrap re-reads the config. Clear - // HEALTHCHECK_ALIVE_URL/BACKUP_URL ONLY on a genuine mode change (parity with the - // TUI's ApplyInstallData) so a same-mode re-run keeps the user's own self URLs and - // an abort never leaves the monitor without a ping target; the off branch also - // writes HEALTHCHECK_MODE=off so no stale MODE lingers (F10-08). - prevMode := installer.DeriveInstallWizardPrefill(template).HealthcheckMode - clearHCURLs := func() { - if hcMode != prevMode { - template = setEnvValue(template, "HEALTHCHECK_ALIVE_URL", "") - template = setEnvValue(template, "HEALTHCHECK_BACKUP_URL", "") - } - } - switch hcMode { - case "self": - template = setEnvValue(template, "HEALTHCHECK_ENABLED", "true") - template = setEnvValue(template, "HEALTHCHECK_MODE", "self") - clearHCURLs() - case "centralized": - template = setEnvValue(template, "HEALTHCHECK_ENABLED", "true") - template = setEnvValue(template, "HEALTHCHECK_MODE", "centralized") - clearHCURLs() - default: // "off" - template = setEnvValue(template, "HEALTHCHECK_ENABLED", "false") - template = setEnvValue(template, "HEALTHCHECK_MODE", "off") - clearHCURLs() - } - if err := installer.WriteConfigFileAtomic(configPath, tmpConfigPath, template); err != nil { - return installConfigResult{}, err - } - - if bootstrap != nil { - bootstrap.Info("✓ Configuration saved at %s", configPath) - } - - return result, nil + return data, nil } func runEncryptionSetupIfNeeded(ctx context.Context, configPath string, enableEncryption, skipConfigWizard bool, bootstrap *logging.BootstrapLogger) (err error) { @@ -728,13 +820,31 @@ func printInstallBanner(configPath string) { fmt.Printf("Configuration file: %s\n\n", configPath) } -func prepareBaseTemplate(ctx context.Context, reader *bufio.Reader, configPath string, bootstrap *logging.BootstrapLogger) (string, bool, bool, error) { +// installWizardBase carries the two views of the wizard's starting template that +// must not be confused. Raw is the base exactly as installer.ResolveExistingConfigDecision +// produced it ("" = no existing file): installer.ApplyInstallData derives +// editingExisting from it and substitutes the embedded default when it is blank, +// so the engine must receive THIS value. Prompt is Raw expanded through +// installer.BaseTemplateOrDefault off the Edit path, and is what the CLI reads its +// own prompt defaults from. +// +// Off the Edit path Raw is provably always "" (installer.ResolveExistingConfigDecision +// returns an empty BaseTemplate for Overwrite/KeepContinue/Cancel and +// adoptCronRunTimeIntoBase returns it unchanged when !FromExistingFile), which is +// what makes the expansion below lossless. Pinned by +// TestPrepareBaseTemplateRawIsEmptyOffTheEditPath. +type installWizardBase struct { + Raw string + Prompt string +} + +func prepareBaseTemplate(ctx context.Context, reader *bufio.Reader, configPath string, bootstrap *logging.BootstrapLogger) (installWizardBase, bool, bool, error) { decision, err := prepareExistingConfigDecisionCLI(ctx, reader, configPath) if err != nil { - return "", false, false, err + return installWizardBase{}, false, false, err } if decision.AbortInstall { - return "", false, false, errInteractiveAborted + return installWizardBase{}, false, false, errInteractiveAborted } // This install is about to rewrite the proxsave cron line FROM the config // (buildInstallCronSchedule) and may hand the schedule to the daemon @@ -751,7 +861,7 @@ func prepareBaseTemplate(ctx context.Context, reader *bufio.Reader, configPath s decision.BaseTemplate = adoptCronRunTimeIntoBase(ctx, decision, configPath, bootstrap) if decision.SkipConfigWizard { fmt.Println("Existing configuration detected, keeping current backup.env and skipping configuration wizard.") - return "", true, false, nil + return installWizardBase{}, true, false, nil } // The shared decision carries the RAW base ("" = embedded default) because // ApplyInstallData derives editingExisting from it. The CLI wizard, unlike the @@ -759,121 +869,146 @@ func prepareBaseTemplate(ctx context.Context, reader *bufio.Reader, configPath s // here - but ONLY off the Edit path: expanding a blank existing backup.env would // rewrite it as the full embedded template instead of the minimal key set it // produces today. Pinned by TestPrepareBaseTemplateEditBlankBaseStaysRaw. - base := decision.BaseTemplate + base := installWizardBase{Raw: decision.BaseTemplate, Prompt: decision.BaseTemplate} if !decision.FromExistingFile { - base = installer.BaseTemplateOrDefault(base) + base.Prompt = installer.BaseTemplateOrDefault(base.Raw) } return base, false, decision.FromExistingFile, nil } -func configureSecondaryStorage(ctx context.Context, reader *bufio.Reader, template string) (string, error) { +// promptSecondaryStorage asks the secondary-storage questions and RETURNS the +// answers; writing them into the template is the engine's job +// (installer.ApplyInstallData -> config.ApplySecondaryStorageSettings). Prompt +// text, ordering, the sanitizeEnvValue placement and both re-prompt loops are +// unchanged from the template-threading version this replaced. +func promptSecondaryStorage(ctx context.Context, reader *bufio.Reader, prefill installer.InstallWizardPrefill) (bool, string, string, error) { fmt.Println("\n--- Secondary storage ---") fmt.Println("Configure an additional local path for redundant copies.") fmt.Println("IMPORTANT: Secondary path must be a filesystem-mounted directory (e.g., /mnt/nas-backup)") fmt.Println("Network shares must be mounted BEFORE running this backup tool.") fmt.Println("For direct network access without mounting, use cloud storage (rclone) instead.") fmt.Println("(You can change these settings later in backup.env)") - prefill := installer.DeriveInstallWizardPrefill(template) enableSecondary, err := confirmDefault(ctx, reader, "Enable secondary backup path?", prefill.SecondaryEnabled) if err != nil { - return "", err + return false, "", "", err } - if enableSecondary { - var secondaryPath string - for { - secondaryPath, err = promptNonEmptyWithDefault(ctx, reader, "Secondary backup path (SECONDARY_PATH): ", prefill.SecondaryPath) - if err != nil { - return "", err - } - secondaryPath = sanitizeEnvValue(secondaryPath) - if err := config.ValidateRequiredSecondaryPath(secondaryPath); err != nil { - fmt.Printf("%v\n", err) - continue - } - break + if !enableSecondary { + return false, "", "", nil + } + var secondaryPath string + for { + secondaryPath, err = promptNonEmptyWithDefault(ctx, reader, "Secondary backup path (SECONDARY_PATH): ", prefill.SecondaryPath) + if err != nil { + return false, "", "", err } - var secondaryLog string - for { - secondaryLog, err = promptOptionalWithDefault(ctx, reader, "Secondary log path (SECONDARY_LOG_PATH, optional - press Enter to skip): ", prefill.SecondaryLogPath) - if err != nil { - return "", err - } - secondaryLog = sanitizeEnvValue(secondaryLog) - if err := config.ValidateOptionalSecondaryLogPath(secondaryLog); err != nil { - fmt.Printf("%v\n", err) - continue - } - break + secondaryPath = sanitizeEnvValue(secondaryPath) + if err := config.ValidateRequiredSecondaryPath(secondaryPath); err != nil { + fmt.Printf("%v\n", err) + continue } - template = config.ApplySecondaryStorageSettings(template, true, secondaryPath, secondaryLog) - } else { - template = config.ApplySecondaryStorageSettings(template, false, "", "") + break + } + var secondaryLog string + for { + secondaryLog, err = promptOptionalWithDefault(ctx, reader, "Secondary log path (SECONDARY_LOG_PATH, optional - press Enter to skip): ", prefill.SecondaryLogPath) + if err != nil { + return false, "", "", err + } + secondaryLog = sanitizeEnvValue(secondaryLog) + if err := config.ValidateOptionalSecondaryLogPath(secondaryLog); err != nil { + fmt.Printf("%v\n", err) + continue + } + break } - return template, nil + return true, secondaryPath, secondaryLog, nil } -func configureCloudStorage(ctx context.Context, reader *bufio.Reader, template string) (string, error) { +// promptCloudStorage asks the cloud-storage questions and RETURNS the answers. +// +// The only behavioral addition over the template-threading version it replaces is +// the post-sanitize re-prompt: promptNonEmptyWithDefault only guarantees the RAW +// answer is non-empty, but sanitizeEnvValue strips NUL/CR/LF, so a control-character +// answer used to write CLOUD_ENABLED=true with an empty CLOUD_REMOTE. That payload +// is exactly what installer.ApplyInstallData's validateCloudInstallData rejects, so +// without this guard an install that used to (wrongly) succeed would now fail at the +// very end of the wizard. Re-prompting reuses promptNonEmpty's own message verbatim. +// +// It does NOT make every previously-succeeding path still succeed, and the earlier +// claim that it did was wrong: the re-prompt consumes an extra input line, so a +// finite piped answer stream that used to complete can now hit EOF and abort with +// nothing written. That is accepted because the alternative is worse -- the old +// behavior wrote CLOUD_ENABLED=true with an empty CLOUD_REMOTE and shifted the next +// scripted answer into the following slot. Reachable only with control characters in +// the answer, i.e. a paste accident or a CRLF-mangled answer file. +func promptCloudStorage(ctx context.Context, reader *bufio.Reader, prefill installer.InstallWizardPrefill) (bool, string, string, error) { fmt.Println("\n--- Cloud storage (rclone) ---") fmt.Println("Remember to configure rclone manually before enabling cloud backups.") - prefill := installer.DeriveInstallWizardPrefill(template) enableCloud, err := confirmDefault(ctx, reader, "Enable cloud backups?", prefill.CloudEnabled) if err != nil { - return "", err + return false, "", "", err } - if enableCloud { - remote, err := promptNonEmptyWithDefault(ctx, reader, "Rclone remote for backups (e.g. myremote:pbs-backups): ", prefill.CloudRemote) + if !enableCloud { + return false, "", "", nil + } + remote, err := promptSanitizedNonEmptyWithDefault(ctx, reader, "Rclone remote for backups (e.g. myremote:pbs-backups): ", prefill.CloudRemote) + if err != nil { + return false, "", "", err + } + logRemote, err := promptSanitizedNonEmptyWithDefault(ctx, reader, "Rclone remote for logs (e.g. myremote:/logs): ", prefill.CloudLogPath) + if err != nil { + return false, "", "", err + } + return true, remote, logRemote, nil +} + +// promptSanitizedNonEmptyWithDefault is promptNonEmptyWithDefault whose +// non-emptiness guarantee survives sanitizeEnvValue (see promptCloudStorage). +// +// The DEFAULT is sanitized too, and that is not cosmetic: a stored CLOUD_REMOTE +// made of control characters comes back from DeriveInstallWizardPrefill unchanged, +// and offering it as an acceptable default would make pressing Enter fail the +// sanitize check forever -- an inescapable loop that ends in an aborted install. +// Sanitizing it degrades the prompt to "no default", which the operator can always +// satisfy by typing a value. +// +// This re-prompt is the one intentional behavior delta of the wizard refactor. It +// does NOT preserve the old behavior on every path: it consumes an extra input +// line, so a finite piped answer stream that used to complete can now hit EOF and +// abort without writing anything. It is preferred anyway because the alternative +// is worse -- the old code wrote CLOUD_ENABLED=true with an empty CLOUD_REMOTE and +// shifted the next answer into the wrong slot. +func promptSanitizedNonEmptyWithDefault(ctx context.Context, reader *bufio.Reader, question, def string) (string, error) { + def = sanitizeEnvValue(def) + for { + raw, err := promptNonEmptyWithDefault(ctx, reader, question, def) if err != nil { return "", err } - remote = sanitizeEnvValue(remote) - logRemote, err := promptNonEmptyWithDefault(ctx, reader, "Rclone remote for logs (e.g. myremote:/logs): ", prefill.CloudLogPath) - if err != nil { - return "", err + if value := sanitizeEnvValue(raw); value != "" { + return value, nil } - logRemote = sanitizeEnvValue(logRemote) - template = setEnvValue(template, "CLOUD_ENABLED", "true") - template = setEnvValue(template, "CLOUD_REMOTE", remote) - template = setEnvValue(template, "CLOUD_LOG_PATH", logRemote) - } else { - template = setEnvValue(template, "CLOUD_ENABLED", "false") - template = setEnvValue(template, "CLOUD_REMOTE", "") - template = setEnvValue(template, "CLOUD_LOG_PATH", "") + fmt.Println("Value cannot be empty.") } - return template, nil } -func configureFirewallRules(ctx context.Context, reader *bufio.Reader, template string) (string, error) { +// promptFirewallRules asks the firewall question and RETURNS the answer. +func promptFirewallRules(ctx context.Context, reader *bufio.Reader, prefill installer.InstallWizardPrefill) (bool, error) { fmt.Println("\n--- Firewall rules ---") fmt.Println("Enable collection of firewall rules (e.g., iptables/nftables).") fmt.Println("(You can change this later in backup.env via BACKUP_FIREWALL_RULES)") - enable, err := confirmDefault(ctx, reader, "Backup firewall rules?", installer.DeriveInstallWizardPrefill(template).FirewallEnabled) - if err != nil { - return "", err - } - if enable { - template = setEnvValue(template, "BACKUP_FIREWALL_RULES", "true") - } else { - template = setEnvValue(template, "BACKUP_FIREWALL_RULES", "false") - } - return template, nil + return confirmDefault(ctx, reader, "Backup firewall rules?", prefill.FirewallEnabled) } -func configureNotifications(ctx context.Context, reader *bufio.Reader, template string) (string, error) { - prefill := installer.DeriveInstallWizardPrefill(template) +// promptNotifications asks the Telegram + email questions (including the delivery +// method when email is enabled) and RETURNS the answers. The BOT_TELEGRAM_TYPE +// seeding, the EMAIL_FALLBACK_PMF removal and the EMAIL_FALLBACK_SENDMAIL write +// this used to perform inline are installer.ApplyInstallData's job. +func promptNotifications(ctx context.Context, reader *bufio.Reader, prefill installer.InstallWizardPrefill) (bool, bool, string, error) { fmt.Println("\n--- Telegram ---") enableTelegram, err := confirmDefault(ctx, reader, "Enable Telegram notifications (centralized)?", prefill.TelegramEnabled) if err != nil { - return "", err - } - if enableTelegram { - template = setEnvValue(template, "TELEGRAM_ENABLED", "true") - // Preserve a stored bot mode (e.g. personal); only seed the centralized - // default when none is set yet, mirroring the TUI's ApplyInstallData. - if strings.TrimSpace(prefill.TelegramType) == "" { - template = setEnvValue(template, "BOT_TELEGRAM_TYPE", "centralized") - } - } else { - template = setEnvValue(template, "TELEGRAM_ENABLED", "false") + return false, false, "", err } fmt.Println("\n--- Email ---") @@ -881,21 +1016,16 @@ func configureNotifications(ctx context.Context, reader *bufio.Reader, template fmt.Println("ProxSave does not collect raw SMTP settings; choose pmf only when Proxmox Notifications is configured.") enableEmail, err := confirmDefault(ctx, reader, "Enable email notifications?", prefill.EmailEnabled) if err != nil { - return "", err + return false, false, "", err } - if enableEmail { - method, err := promptEmailDeliveryMethod(ctx, reader, prefill.EmailDeliveryMethod) - if err != nil { - return "", err - } - template = setEnvValue(template, "EMAIL_ENABLED", "true") - template = setEnvValue(template, "EMAIL_DELIVERY_METHOD", method) - template = installer.UnsetEnvValueInTemplate(template, "EMAIL_FALLBACK_PMF") - template = setEnvValue(template, "EMAIL_FALLBACK_SENDMAIL", "true") - } else { - template = setEnvValue(template, "EMAIL_ENABLED", "false") + if !enableEmail { + return enableTelegram, false, "", nil + } + method, err := promptEmailDeliveryMethod(ctx, reader, prefill.EmailDeliveryMethod) + if err != nil { + return false, false, "", err } - return template, nil + return enableTelegram, true, method, nil } func promptEmailDeliveryMethod(ctx context.Context, reader *bufio.Reader, defaultMethod string) (string, error) { @@ -926,18 +1056,12 @@ func promptEmailDeliveryMethod(ctx context.Context, reader *bufio.Reader, defaul } } -func configureEncryption(ctx context.Context, reader *bufio.Reader, template *string) (bool, error) { +// promptEncryption asks the encryption question and RETURNS the answer. The +// *string out-param it used to take existed only to write ENCRYPT_ARCHIVE, which +// installer.ApplyInstallData now owns. +func promptEncryption(ctx context.Context, reader *bufio.Reader, prefill installer.InstallWizardPrefill) (bool, error) { fmt.Println("\n--- Encryption ---") - enableEncryption, err := confirmDefault(ctx, reader, "Enable backup encryption?", installer.DeriveInstallWizardPrefill(*template).EncryptionEnabled) - if err != nil { - return false, err - } - if enableEncryption { - *template = setEnvValue(*template, "ENCRYPT_ARCHIVE", "true") - } else { - *template = setEnvValue(*template, "ENCRYPT_ARCHIVE", "false") - } - return enableEncryption, nil + return confirmDefault(ctx, reader, "Enable backup encryption?", prefill.EncryptionEnabled) } // schedulerEngineDefault picks the engine prompt default. Fresh installs and diff --git a/cmd/proxsave/install_test.go b/cmd/proxsave/install_test.go index 8ee4af7a..73412709 100644 --- a/cmd/proxsave/install_test.go +++ b/cmd/proxsave/install_test.go @@ -9,7 +9,9 @@ import ( "strings" "testing" + "github.com/tis24dev/proxsave/internal/config" cronutil "github.com/tis24dev/proxsave/internal/cron" + "github.com/tis24dev/proxsave/internal/installer" "github.com/tis24dev/proxsave/internal/logging" ) @@ -265,11 +267,11 @@ func TestResetInstallBaseDirWithContext_CanceledBeforeRemoval(t *testing.T) { func TestPrepareBaseTemplateExistingSkip(t *testing.T) { cfgFile := createTempFile(t, "existing config") reader := bufio.NewReader(strings.NewReader("3\n")) - var tmpl string + var base installWizardBase var skip bool var err error captureStdout(t, func() { - tmpl, skip, _, err = prepareBaseTemplate(context.Background(), reader, cfgFile, nil) + base, skip, _, err = prepareBaseTemplate(context.Background(), reader, cfgFile, nil) }) if err != nil { t.Fatalf("prepareBaseTemplate error: %v", err) @@ -277,19 +279,19 @@ func TestPrepareBaseTemplateExistingSkip(t *testing.T) { if !skip { t.Fatalf("expected skip when user declines overwrite") } - if tmpl != "" { - t.Fatalf("template should be empty when skipping wizard") + if base.Prompt != "" || base.Raw != "" { + t.Fatalf("template should be empty when skipping wizard, got %+v", base) } } func TestPrepareBaseTemplateOverwrite(t *testing.T) { cfgFile := createTempFile(t, "old") reader := bufio.NewReader(strings.NewReader("1\n")) - var tmpl string + var base installWizardBase var skip bool var err error captureStdout(t, func() { - tmpl, skip, _, err = prepareBaseTemplate(context.Background(), reader, cfgFile, nil) + base, skip, _, err = prepareBaseTemplate(context.Background(), reader, cfgFile, nil) }) if err != nil { t.Fatalf("prepareBaseTemplate error: %v", err) @@ -297,7 +299,7 @@ func TestPrepareBaseTemplateOverwrite(t *testing.T) { if skip { t.Fatalf("expected skip=false after overwrite confirmation") } - if tmpl == "" { + if base.Prompt == "" { t.Fatalf("expected template contents") } } @@ -305,11 +307,11 @@ func TestPrepareBaseTemplateOverwrite(t *testing.T) { func TestPrepareBaseTemplateEditExisting(t *testing.T) { cfgFile := createTempFile(t, "EXISTING=1\n") reader := bufio.NewReader(strings.NewReader("2\n")) - var tmpl string + var base installWizardBase var skip bool var err error captureStdout(t, func() { - tmpl, skip, _, err = prepareBaseTemplate(context.Background(), reader, cfgFile, nil) + base, skip, _, err = prepareBaseTemplate(context.Background(), reader, cfgFile, nil) }) if err != nil { t.Fatalf("prepareBaseTemplate error: %v", err) @@ -317,8 +319,8 @@ func TestPrepareBaseTemplateEditExisting(t *testing.T) { if skip { t.Fatalf("expected skip=false for edit existing") } - if !strings.Contains(tmpl, "EXISTING=1") { - t.Fatalf("expected existing template content, got %q", tmpl) + if !strings.Contains(base.Prompt, "EXISTING=1") { + t.Fatalf("expected existing template content, got %q", base.Prompt) } } @@ -331,170 +333,198 @@ func TestPrepareBaseTemplateCancel(t *testing.T) { } } -func TestConfigureSecondaryStorageEnabled(t *testing.T) { - var result string +func TestPromptSecondaryStorageEnabled(t *testing.T) { + var enabled bool + var path, logPath string var err error ctx := context.Background() reader := bufio.NewReader(strings.NewReader("y\n/mnt/secondary\n/mnt/secondary/log\n")) captureStdout(t, func() { - result, err = configureSecondaryStorage(ctx, reader, "") + enabled, path, logPath, err = promptSecondaryStorage(ctx, reader, installer.DeriveInstallWizardPrefill("")) }) if err != nil { - t.Fatalf("configureSecondaryStorage error: %v", err) + t.Fatalf("promptSecondaryStorage error: %v", err) } - if !strings.Contains(result, "SECONDARY_ENABLED=true") { - t.Fatalf("expected SECONDARY_ENABLED=true in template: %q", result) + if !enabled { + t.Fatal("expected secondary storage enabled") } - if !strings.Contains(result, "SECONDARY_PATH=/mnt/secondary") { - t.Fatalf("expected secondary path in template: %q", result) + if path != "/mnt/secondary" { + t.Fatalf("secondary path = %q, want /mnt/secondary", path) } - if !strings.Contains(result, "SECONDARY_LOG_PATH=/mnt/secondary/log") { - t.Fatalf("expected secondary log path in template: %q", result) + if logPath != "/mnt/secondary/log" { + t.Fatalf("secondary log path = %q, want /mnt/secondary/log", logPath) } } -func TestConfigureSecondaryStorageEnabledWithEmptyLogPath(t *testing.T) { - var result string +func TestPromptSecondaryStorageEnabledWithEmptyLogPath(t *testing.T) { + var enabled bool + var path, logPath string var err error ctx := context.Background() reader := bufio.NewReader(strings.NewReader("y\n/mnt/secondary\n\n")) captureStdout(t, func() { - result, err = configureSecondaryStorage(ctx, reader, "") + enabled, path, logPath, err = promptSecondaryStorage(ctx, reader, installer.DeriveInstallWizardPrefill("")) }) if err != nil { - t.Fatalf("configureSecondaryStorage error: %v", err) + t.Fatalf("promptSecondaryStorage error: %v", err) } - if !strings.Contains(result, "SECONDARY_ENABLED=true") { - t.Fatalf("expected SECONDARY_ENABLED=true in template: %q", result) + if !enabled { + t.Fatal("expected secondary storage enabled") } - if !strings.Contains(result, "SECONDARY_PATH=/mnt/secondary") { - t.Fatalf("expected secondary path in template: %q", result) + if path != "/mnt/secondary" { + t.Fatalf("secondary path = %q, want /mnt/secondary", path) } - if !strings.Contains(result, "SECONDARY_LOG_PATH=") { - t.Fatalf("expected empty secondary log path in template: %q", result) + if logPath != "" { + t.Fatalf("secondary log path = %q, want empty", logPath) } } -func TestConfigureSecondaryStorageRejectsInvalidBackupPath(t *testing.T) { - var result string +func TestPromptSecondaryStorageRejectsInvalidBackupPath(t *testing.T) { + var path string var err error ctx := context.Background() reader := bufio.NewReader(strings.NewReader("y\nrelative/path\n/mnt/secondary\n\n")) captureStdout(t, func() { - result, err = configureSecondaryStorage(ctx, reader, "") + _, path, _, err = promptSecondaryStorage(ctx, reader, installer.DeriveInstallWizardPrefill("")) }) if err != nil { - t.Fatalf("configureSecondaryStorage error: %v", err) + t.Fatalf("promptSecondaryStorage error: %v", err) } - if !strings.Contains(result, "SECONDARY_PATH=/mnt/secondary") { - t.Fatalf("expected corrected secondary path in template: %q", result) + if path != "/mnt/secondary" { + t.Fatalf("expected corrected secondary path, got %q", path) } } -func TestConfigureSecondaryStorageRejectsInvalidLogPath(t *testing.T) { - var result string +func TestPromptSecondaryStorageRejectsInvalidLogPath(t *testing.T) { + var logPath string var err error ctx := context.Background() reader := bufio.NewReader(strings.NewReader("y\n/mnt/secondary\nremote:/logs\n\n")) captureStdout(t, func() { - result, err = configureSecondaryStorage(ctx, reader, "") + _, _, logPath, err = promptSecondaryStorage(ctx, reader, installer.DeriveInstallWizardPrefill("")) }) if err != nil { - t.Fatalf("configureSecondaryStorage error: %v", err) + t.Fatalf("promptSecondaryStorage error: %v", err) } - if !strings.Contains(result, "SECONDARY_LOG_PATH=") { - t.Fatalf("expected empty secondary log path in template: %q", result) + if logPath != "" { + t.Fatalf("expected empty secondary log path, got %q", logPath) } } -func TestConfigureSecondaryStorageDisabled(t *testing.T) { - var result string +func TestPromptSecondaryStorageDisabled(t *testing.T) { + var enabled bool + var path, logPath string var err error ctx := context.Background() reader := bufio.NewReader(strings.NewReader("n\n")) captureStdout(t, func() { - result, err = configureSecondaryStorage(ctx, reader, "") + enabled, path, logPath, err = promptSecondaryStorage(ctx, reader, installer.DeriveInstallWizardPrefill("")) }) if err != nil { - t.Fatalf("configureSecondaryStorage error: %v", err) - } - if !strings.Contains(result, "SECONDARY_ENABLED=false") { - t.Fatalf("expected disabled flag in template: %q", result) + t.Fatalf("promptSecondaryStorage error: %v", err) } - if !strings.Contains(result, "SECONDARY_PATH=") { - t.Fatalf("expected cleared secondary path in template: %q", result) + if enabled { + t.Fatal("expected secondary storage disabled") } - if !strings.Contains(result, "SECONDARY_LOG_PATH=") { - t.Fatalf("expected cleared secondary log path in template: %q", result) + if path != "" || logPath != "" { + t.Fatalf("declining must clear both paths, got path=%q log=%q", path, logPath) } } -func TestConfigureSecondaryStorageDisabledClearsExistingValues(t *testing.T) { - var result string +func TestPromptSecondaryStorageDisabledClearsExistingValues(t *testing.T) { + var enabled bool + var path, logPath string var err error ctx := context.Background() reader := bufio.NewReader(strings.NewReader("n\n")) template := "SECONDARY_ENABLED=true\nSECONDARY_PATH=/mnt/old-secondary\nSECONDARY_LOG_PATH=/mnt/old-secondary/logs\n" captureStdout(t, func() { - result, err = configureSecondaryStorage(ctx, reader, template) + enabled, path, logPath, err = promptSecondaryStorage(ctx, reader, installer.DeriveInstallWizardPrefill(template)) }) if err != nil { - t.Fatalf("configureSecondaryStorage error: %v", err) + t.Fatalf("promptSecondaryStorage error: %v", err) } - for _, needle := range []string{ - "SECONDARY_ENABLED=false", - "SECONDARY_PATH=", - "SECONDARY_LOG_PATH=", - } { - if !strings.Contains(result, needle) { - t.Fatalf("expected %q in template: %q", needle, result) - } + if enabled { + t.Fatal("expected secondary storage disabled") } - if strings.Contains(result, "/mnt/old-secondary") { - t.Fatalf("expected old secondary values to be cleared: %q", result) + // The stored values must not survive the decline: the payload carries empty + // paths, and config.ApplySecondaryStorageSettings clears both keys + // (pinned by internal/config env_mutation_test.go). + if path != "" || logPath != "" { + t.Fatalf("expected old secondary values to be cleared, got path=%q log=%q", path, logPath) } } -func TestConfigureCloudStorageEnabled(t *testing.T) { - var result string +func TestPromptCloudStorageEnabled(t *testing.T) { + var enabled bool + var remote, logRemote string var err error ctx := context.Background() reader := bufio.NewReader(strings.NewReader("y\nremote:pbs\nremote:/logs\n")) captureStdout(t, func() { - result, err = configureCloudStorage(ctx, reader, "") + enabled, remote, logRemote, err = promptCloudStorage(ctx, reader, installer.DeriveInstallWizardPrefill("")) }) if err != nil { - t.Fatalf("configureCloudStorage error: %v", err) + t.Fatalf("promptCloudStorage error: %v", err) } - if !strings.Contains(result, "CLOUD_ENABLED=true") { - t.Fatalf("expected enabled flag: %q", result) + if !enabled { + t.Fatal("expected cloud storage enabled") } - if !strings.Contains(result, "CLOUD_REMOTE=remote:pbs") { - t.Fatalf("expected remote entry: %q", result) + if remote != "remote:pbs" { + t.Fatalf("remote = %q, want remote:pbs", remote) } - if !strings.Contains(result, "CLOUD_LOG_PATH=remote:/logs") { - t.Fatalf("expected log remote entry: %q", result) + if logRemote != "remote:/logs" { + t.Fatalf("log remote = %q, want remote:/logs", logRemote) } } -func TestConfigureCloudStorageDisabled(t *testing.T) { - var result string +func TestPromptCloudStorageDisabled(t *testing.T) { + var enabled bool + var remote, logRemote string var err error ctx := context.Background() reader := bufio.NewReader(strings.NewReader("n\n")) captureStdout(t, func() { - result, err = configureCloudStorage(ctx, reader, "") + enabled, remote, logRemote, err = promptCloudStorage(ctx, reader, installer.DeriveInstallWizardPrefill("")) }) if err != nil { - t.Fatalf("configureCloudStorage error: %v", err) + t.Fatalf("promptCloudStorage error: %v", err) } - if !strings.Contains(result, "CLOUD_ENABLED=false") { - t.Fatalf("expected disabled flag: %q", result) + if enabled { + t.Fatal("expected cloud storage disabled") + } + if remote != "" || logRemote != "" { + t.Fatalf("declining must clear both remotes, got remote=%q log=%q", remote, logRemote) + } +} + +// TestPromptCloudStorageRejectsValueEmptiedBySanitize pins the guard that keeps +// installer.ApplyInstallData's validateCloudInstallData unreachable: an answer that +// is non-empty before sanitizeEnvValue but empty after it must re-prompt instead of +// producing CLOUD_ENABLED=true with an empty remote. +func TestPromptCloudStorageRejectsValueEmptiedBySanitize(t *testing.T) { + var enabled bool + var remote, logRemote string + var err error + ctx := context.Background() + reader := bufio.NewReader(strings.NewReader("y\n\x00\nremote:pbs\nremote:/logs\n")) + output := captureStdout(t, func() { + enabled, remote, logRemote, err = promptCloudStorage(ctx, reader, installer.DeriveInstallWizardPrefill("")) + }) + if err != nil { + t.Fatalf("promptCloudStorage error: %v", err) + } + if !enabled || remote != "remote:pbs" || logRemote != "remote:/logs" { + t.Fatalf("enabled=%v remote=%q log=%q", enabled, remote, logRemote) + } + if !strings.Contains(output, "Value cannot be empty.") { + t.Fatalf("expected the empty-value re-prompt message, got %q", output) } } -func TestConfigureCloudStorageKeepsExistingOnEdit(t *testing.T) { - var result string +func TestPromptCloudStorageKeepsExistingOnEdit(t *testing.T) { + var enabled bool + var remote, logRemote string var err error ctx := context.Background() // Pressing Enter through every prompt while editing an existing config must @@ -502,120 +532,124 @@ func TestConfigureCloudStorageKeepsExistingOnEdit(t *testing.T) { reader := bufio.NewReader(strings.NewReader("\n\n\n")) template := "CLOUD_ENABLED=true\nCLOUD_REMOTE=remote:pbs\nCLOUD_LOG_PATH=remote:/logs\n" captureStdout(t, func() { - result, err = configureCloudStorage(ctx, reader, template) + enabled, remote, logRemote, err = promptCloudStorage(ctx, reader, installer.DeriveInstallWizardPrefill(template)) }) if err != nil { - t.Fatalf("configureCloudStorage error: %v", err) + t.Fatalf("promptCloudStorage error: %v", err) } - for _, want := range []string{ - "CLOUD_ENABLED=true", - "CLOUD_REMOTE=remote:pbs", - "CLOUD_LOG_PATH=remote:/logs", - } { - if !strings.Contains(result, want) { - t.Fatalf("expected %q preserved on no-op edit, got: %q", want, result) - } + if !enabled { + t.Fatal("a no-op edit must keep cloud storage enabled") + } + if remote != "remote:pbs" || logRemote != "remote:/logs" { + t.Fatalf("expected stored remotes preserved on no-op edit, got remote=%q log=%q", remote, logRemote) } } -func TestConfigureFirewallRulesDefaultsToDisabled(t *testing.T) { - var result string +func TestPromptFirewallRulesDefaultsToDisabled(t *testing.T) { + var enabled bool var err error ctx := context.Background() reader := bufio.NewReader(strings.NewReader("\n")) captureStdout(t, func() { - result, err = configureFirewallRules(ctx, reader, "") + enabled, err = promptFirewallRules(ctx, reader, installer.DeriveInstallWizardPrefill("")) }) if err != nil { - t.Fatalf("configureFirewallRules error: %v", err) + t.Fatalf("promptFirewallRules error: %v", err) } - if !strings.Contains(result, "BACKUP_FIREWALL_RULES=false") { - t.Fatalf("expected disabled flag: %q", result) + if enabled { + t.Fatal("expected firewall rules disabled by default") } } -func TestConfigureFirewallRulesDisabled(t *testing.T) { - var result string +func TestPromptFirewallRulesDisabled(t *testing.T) { + var enabled bool var err error ctx := context.Background() reader := bufio.NewReader(strings.NewReader("n\n")) captureStdout(t, func() { - result, err = configureFirewallRules(ctx, reader, "") + enabled, err = promptFirewallRules(ctx, reader, installer.DeriveInstallWizardPrefill("")) }) if err != nil { - t.Fatalf("configureFirewallRules error: %v", err) + t.Fatalf("promptFirewallRules error: %v", err) } - if !strings.Contains(result, "BACKUP_FIREWALL_RULES=false") { - t.Fatalf("expected disabled flag: %q", result) + if enabled { + t.Fatal("expected firewall rules disabled") } } -func TestConfigureNotifications(t *testing.T) { - var result string +func TestPromptNotifications(t *testing.T) { + var telegram, email bool + var method string var err error ctx := context.Background() reader := bufio.NewReader(strings.NewReader("y\nn\n")) captureStdout(t, func() { - result, err = configureNotifications(ctx, reader, "") + telegram, email, method, err = promptNotifications(ctx, reader, installer.DeriveInstallWizardPrefill("")) }) if err != nil { - t.Fatalf("configureNotifications error: %v", err) + t.Fatalf("promptNotifications error: %v", err) + } + if !telegram { + t.Fatal("expected telegram enabled") } - if !strings.Contains(result, "TELEGRAM_ENABLED=true") { - t.Fatalf("expected telegram enabled in template: %q", result) + if email { + t.Fatal("expected email disabled") } - if !strings.Contains(result, "EMAIL_ENABLED=false") { - t.Fatalf("expected email disabled in template: %q", result) + if method != "" { + t.Fatalf("declining email must not collect a delivery method, got %q", method) } } -func TestConfigureNotificationsEmailDefaultsToRelaySendmailFallback(t *testing.T) { - var result string +func TestPromptNotificationsEmailDefaultsToRelay(t *testing.T) { + var telegram, email bool + var method string var err error ctx := context.Background() reader := bufio.NewReader(strings.NewReader("n\ny\n\n")) captureStdout(t, func() { - result, err = configureNotifications(ctx, reader, "") + telegram, email, method, err = promptNotifications(ctx, reader, installer.DeriveInstallWizardPrefill("")) }) if err != nil { - t.Fatalf("configureNotifications error: %v", err) + t.Fatalf("promptNotifications error: %v", err) } - for _, want := range []string{ - "TELEGRAM_ENABLED=false", - "EMAIL_ENABLED=true", - "EMAIL_DELIVERY_METHOD=relay", - "EMAIL_FALLBACK_SENDMAIL=true", - } { - if !strings.Contains(result, want) { - t.Fatalf("missing %q in template: %q", want, result) - } + if telegram { + t.Fatal("expected telegram disabled") } + if !email { + t.Fatal("expected email enabled") + } + if method != "relay" { + t.Fatalf("delivery method = %q, want relay", method) + } + // EMAIL_FALLBACK_SENDMAIL=true is written by installer.ApplyInstallData from the + // non-nil EmailFallbackSendmail the collector always sends (pinned by + // TestCollectInstallWizardDataCLIAlwaysSendsNonNilFlags and + // internal/installer/install_data_test.go). } -func TestConfigureNotificationsKeepsExistingOnEdit(t *testing.T) { - var result string +func TestPromptNotificationsKeepsExistingOnEdit(t *testing.T) { + var telegram, email bool + var method string var err error ctx := context.Background() // Enter through telegram, email and the email-method prompts: a no-op edit - // must preserve the stored personal bot mode and pmf delivery method instead - // of clobbering them to centralized/relay. + // must preserve the stored pmf delivery method instead of clobbering it to + // relay. The stored personal bot mode is preserved by installer.ApplyInstallData + // (it only seeds BOT_TELEGRAM_TYPE when the existing config has none), pinned by + // the EditExistingNoOp characterization golden. reader := bufio.NewReader(strings.NewReader("\n\n\n")) template := "TELEGRAM_ENABLED=true\nBOT_TELEGRAM_TYPE=personal\nEMAIL_ENABLED=true\nEMAIL_DELIVERY_METHOD=pmf\n" captureStdout(t, func() { - result, err = configureNotifications(ctx, reader, template) + telegram, email, method, err = promptNotifications(ctx, reader, installer.DeriveInstallWizardPrefill(template)) }) if err != nil { - t.Fatalf("configureNotifications error: %v", err) + t.Fatalf("promptNotifications error: %v", err) } - for _, want := range []string{ - "TELEGRAM_ENABLED=true", - "BOT_TELEGRAM_TYPE=personal", - "EMAIL_ENABLED=true", - "EMAIL_DELIVERY_METHOD=pmf", - } { - if !strings.Contains(result, want) { - t.Fatalf("expected %q preserved on no-op edit, got: %q", want, result) - } + if !telegram || !email { + t.Fatalf("a no-op edit must keep both channels enabled, got telegram=%v email=%v", telegram, email) + } + if method != "pmf" { + t.Fatalf("delivery method = %q, want pmf preserved", method) } } @@ -644,38 +678,33 @@ func TestRunPostInstallAuditCLIAbortIsNonBlocking(t *testing.T) { } } -func TestConfigureEncryption(t *testing.T) { +func TestPromptEncryption(t *testing.T) { var enabled bool var err error - template := "" ctx := context.Background() reader := bufio.NewReader(strings.NewReader("y\n")) captureStdout(t, func() { - enabled, err = configureEncryption(ctx, reader, &template) + enabled, err = promptEncryption(ctx, reader, installer.DeriveInstallWizardPrefill("")) }) if err != nil { - t.Fatalf("configureEncryption error: %v", err) + t.Fatalf("promptEncryption error: %v", err) } if !enabled { t.Fatalf("expected encryption enabled") } - if !strings.Contains(template, "ENCRYPT_ARCHIVE=true") { - t.Fatalf("expected ENCRYPT_ARCHIVE flag, got %q", template) - } + // Declining while the stored config has it enabled must return false (the + // ENCRYPT_ARCHIVE=false write is installer.ApplyInstallData's job). reader = bufio.NewReader(strings.NewReader("n\n")) captureStdout(t, func() { - enabled, err = configureEncryption(ctx, reader, &template) + enabled, err = promptEncryption(ctx, reader, installer.DeriveInstallWizardPrefill("ENCRYPT_ARCHIVE=true\n")) }) if err != nil { - t.Fatalf("configureEncryption disable error: %v", err) + t.Fatalf("promptEncryption disable error: %v", err) } if enabled { t.Fatalf("expected disabled encryption") } - if !strings.Contains(template, "ENCRYPT_ARCHIVE=false") { - t.Fatalf("expected disabled flag") - } } func TestConfigureCronTime(t *testing.T) { @@ -889,11 +918,11 @@ func TestPrepareBaseTemplateEditBlankBaseStaysRaw(t *testing.T) { t.Run(tc.name, func(t *testing.T) { cfgFile := createTempFile(t, tc.content) reader := bufio.NewReader(strings.NewReader("2\n")) - var tmpl string + var base installWizardBase var fromExisting bool var err error captureStdout(t, func() { - tmpl, _, fromExisting, err = prepareBaseTemplate(context.Background(), reader, cfgFile, nil) + base, _, fromExisting, err = prepareBaseTemplate(context.Background(), reader, cfgFile, nil) }) if err != nil { t.Fatalf("prepareBaseTemplate error: %v", err) @@ -901,9 +930,457 @@ func TestPrepareBaseTemplateEditBlankBaseStaysRaw(t *testing.T) { if !fromExisting { t.Fatal("edit must report fromExisting=true") } - if tmpl != tc.content { - t.Fatalf("blank base must stay raw; got %d bytes (%q), want the file content back", len(tmpl), tmpl) + if base.Prompt != tc.content { + t.Fatalf("blank base must stay raw; got %d bytes (%q), want the file content back", len(base.Prompt), base.Prompt) + } + if base.Raw != tc.content { + t.Fatalf("blank base must reach the engine raw; got %d bytes (%q), want the file content back", len(base.Raw), base.Raw) + } + }) + } +} + +// stubCrontabDerivation neutralizes the host's real crontab so the Edit path is +// deterministic (adoptCronRunTimeIntoBase would otherwise adopt a run time from it). +func stubCrontabDerivation(t *testing.T) { + t.Helper() + original := deriveSchedulerTimeFromCrontabFn + t.Cleanup(func() { deriveSchedulerTimeFromCrontabFn = original }) + deriveSchedulerTimeFromCrontabFn = func(ctx context.Context, configPath string) schedulerTimeSeed { + return schedulerTimeSeed{} + } +} + +// TestRunConfigWizardCLIHealthcheckModeResult pins installConfigResult.HealthcheckMode, +// which nothing asserted before and which gates runHealthcheckSelfParamsCLI in +// runInstall: getting it wrong silently skips (or wrongly shows) the self-mode +// ping-URL screen. +func TestRunConfigWizardCLIHealthcheckModeResult(t *testing.T) { + tests := []struct { + name string + script string + wantMode string + wantHCPrompt bool + wantScheduler string + }{ + {"daemon self", "n\nn\nn\nn\nn\nn\ndaemon\nself\n03:15\n", "self", true, "daemon"}, + {"daemon off", "n\nn\nn\nn\nn\nn\ndaemon\noff\n03:15\n", "off", true, "daemon"}, + {"cron forces off without a prompt", "n\nn\nn\nn\nn\nn\ncron\n03:15\n", "off", false, "cron"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + configPath := filepath.Join(t.TempDir(), "env", "backup.env") + var result installConfigResult + var err error + output := captureStdout(t, func() { + reader := bufio.NewReader(strings.NewReader(tt.script)) + result, err = runConfigWizardCLI(context.Background(), reader, configPath, configPath+".tmp", "/opt/proxsave", nil) + }) + if err != nil { + t.Fatalf("runConfigWizardCLI error: %v", err) + } + if result.HealthcheckMode != tt.wantMode { + t.Fatalf("HealthcheckMode = %q, want %q", result.HealthcheckMode, tt.wantMode) + } + if result.SchedulerMode != tt.wantScheduler { + t.Fatalf("SchedulerMode = %q, want %q", result.SchedulerMode, tt.wantScheduler) + } + if got := strings.Contains(output, "Healthchecks monitoring:"); got != tt.wantHCPrompt { + t.Fatalf("healthcheck prompt shown = %v, want %v", got, tt.wantHCPrompt) } }) } } + +// TestCollectInstallWizardDataCLIBlankEditKeepsStoredDefaults pins +// wizardBlankBaseStandIn. The scheduler/healthcheck/run-at defaults used to be read +// off the RUNNING template, which the secondary-storage step had already made +// non-blank; feeding a genuinely blank Edit base to the *Default helpers instead +// flips the prompts to [daemon]/[centralized], shows the healthcheck prompt where it +// was never shown, consumes an extra scripted answer and flips HealthcheckMode +// off -> centralized. Deleting the stand-in fails this test. +func TestCollectInstallWizardDataCLIBlankEditKeepsStoredDefaults(t *testing.T) { + stubCrontabDerivation(t) + cfgFile := createTempFile(t, "") + var result installConfigResult + var err error + output := captureStdout(t, func() { + reader := bufio.NewReader(strings.NewReader("2\nn\nn\nn\nn\nn\nn\n\n03:15\n")) + result, err = runConfigWizardCLI(context.Background(), reader, cfgFile, cfgFile+".tmp", "/opt/proxsave", nil) + }) + if err != nil { + t.Fatalf("runConfigWizardCLI error: %v", err) + } + if !strings.Contains(output, "or cron [cron]: ") { + t.Fatalf("blank Edit must keep the [cron] scheduler default, got:\n%s", output) + } + if strings.Contains(output, "Healthchecks monitoring:") { + t.Fatalf("blank Edit defaults to cron, so no healthcheck prompt may appear:\n%s", output) + } + if result.SchedulerMode != "cron" { + t.Fatalf("SchedulerMode = %q, want cron", result.SchedulerMode) + } + if result.HealthcheckMode != "off" { + t.Fatalf("HealthcheckMode = %q, want off", result.HealthcheckMode) + } + + // Same base, but answering daemon: the healthcheck default must stay [off]. + daemonOutput := captureStdout(t, func() { + reader := bufio.NewReader(strings.NewReader("2\nn\nn\nn\nn\nn\nn\ndaemon\n\n03:15\n")) + _, err = runConfigWizardCLI(context.Background(), reader, cfgFile, cfgFile+".tmp", "/opt/proxsave", nil) + }) + if err != nil { + t.Fatalf("runConfigWizardCLI (daemon) error: %v", err) + } + if !strings.Contains(daemonOutput, "off, centralized, or self [off]: ") { + t.Fatalf("blank Edit must keep the [off] healthcheck default, got:\n%s", daemonOutput) + } +} + +// TestRunConfigWizardCLIBlankEditKeepsMinimalKeySet pins the deliberate +// bug-compatibility of wizardBlankEditBaseMarker: editing an empty backup.env still +// writes the ~290-byte minimal key set, NOT the full embedded template that +// installer.ApplyInstallData substitutes for a blank base. Realigning this with the +// Charm front-end is a separate, signed-off behavior change; without the marker this +// test fails and the whole suite - characterization goldens included - stays green. +func TestRunConfigWizardCLIBlankEditKeepsMinimalKeySet(t *testing.T) { + for _, tc := range []struct { + name string + content string + }{ + {"zero byte", ""}, + {"whitespace only", " \n\t\n"}, + } { + t.Run(tc.name, func(t *testing.T) { + stubCrontabDerivation(t) + cfgFile := createTempFile(t, tc.content) + var err error + captureStdout(t, func() { + reader := bufio.NewReader(strings.NewReader("2\nn\nn\nn\nn\nn\nn\n\n03:15\n")) + _, err = runConfigWizardCLI(context.Background(), reader, cfgFile, cfgFile+".tmp", "/opt/proxsave", nil) + }) + if err != nil { + t.Fatalf("runConfigWizardCLI error: %v", err) + } + written, readErr := os.ReadFile(cfgFile) + if readErr != nil { + t.Fatalf("read back: %v", readErr) + } + if strings.Contains(string(written), "BACKUP_ENABLED") { + t.Fatalf("blank Edit must not expand to the embedded template (%d bytes):\n%s", len(written), written) + } + if strings.Contains(string(written), wizardBlankEditBaseMarker) { + t.Fatalf("the blank-base marker must never reach backup.env:\n%s", written) + } + values := parseWrittenEnvForTest(string(written)) + if values["SCHEDULER_MODE"] != "cron" || values["HEALTHCHECK_MODE"] != "off" { + t.Fatalf("unexpected minimal key set:\n%s", written) + } + if !strings.HasPrefix(string(written), tc.content) { + t.Fatalf("the operator's original bytes must survive verbatim:\n%q", string(written)) + } + }) + } +} + +// TestPrepareBaseTemplateRawIsEmptyOffTheEditPath pins the raw/expanded split that +// installWizardBase exists for. It is NOT covered by the characterization goldens: +// handing installer.ApplyInstallData the EXPANDED base instead of the raw one is +// byte-identical on every golden scenario (measured), because the embedded template +// already carries BOT_TELEGRAM_TYPE=centralized and the collector always supplies a +// non-empty EmailDeliveryMethod and a non-nil EmailFallbackSendmail. So this is the +// only thing that keeps the split honest. +func TestPrepareBaseTemplateRawIsEmptyOffTheEditPath(t *testing.T) { + expanded := config.DefaultEnvTemplate() + + t.Run("no existing file", func(t *testing.T) { + stubCrontabDerivation(t) + configPath := filepath.Join(t.TempDir(), "backup.env") + var base installWizardBase + var err error + captureStdout(t, func() { + base, _, _, err = prepareBaseTemplate(context.Background(), bufio.NewReader(strings.NewReader("")), configPath, nil) + }) + if err != nil { + t.Fatalf("prepareBaseTemplate error: %v", err) + } + if base.Raw != "" { + t.Fatalf("fresh install must hand the engine an empty raw base, got %d bytes", len(base.Raw)) + } + if base.Prompt != expanded { + t.Fatalf("fresh install must read prompt defaults from the embedded template") + } + }) + + t.Run("overwrite", func(t *testing.T) { + stubCrontabDerivation(t) + cfgFile := createTempFile(t, "EXISTING=1\n") + var base installWizardBase + var err error + captureStdout(t, func() { + base, _, _, err = prepareBaseTemplate(context.Background(), bufio.NewReader(strings.NewReader("1\n")), cfgFile, nil) + }) + if err != nil { + t.Fatalf("prepareBaseTemplate error: %v", err) + } + if base.Raw != "" { + t.Fatalf("overwrite must hand the engine an empty raw base, got %q", base.Raw) + } + if base.Prompt != expanded { + t.Fatalf("overwrite must read prompt defaults from the embedded template") + } + }) + + t.Run("keep existing", func(t *testing.T) { + stubCrontabDerivation(t) + cfgFile := createTempFile(t, "EXISTING=1\n") + var base installWizardBase + var skip bool + var err error + captureStdout(t, func() { + base, skip, _, err = prepareBaseTemplate(context.Background(), bufio.NewReader(strings.NewReader("3\n")), cfgFile, nil) + }) + if err != nil { + t.Fatalf("prepareBaseTemplate error: %v", err) + } + if !skip { + t.Fatal("expected skip=true") + } + if base.Raw != "" || base.Prompt != "" { + t.Fatalf("keep existing must return a zero base, got %+v", base) + } + }) + + t.Run("edit keeps raw and prompt equal", func(t *testing.T) { + stubCrontabDerivation(t) + const content = "EXISTING=1\n" + cfgFile := createTempFile(t, content) + var base installWizardBase + var err error + captureStdout(t, func() { + base, _, _, err = prepareBaseTemplate(context.Background(), bufio.NewReader(strings.NewReader("2\n")), cfgFile, nil) + }) + if err != nil { + t.Fatalf("prepareBaseTemplate error: %v", err) + } + if base.Raw != content || base.Prompt != content { + t.Fatalf("edit must keep both views raw, got raw=%q prompt=%q", base.Raw, base.Prompt) + } + }) +} + +// TestCollectInstallWizardDataCLIAlwaysSendsNonNilFlags pins two deliberate +// bug-compatibilities, so the follow-up commits that change them fail loudly instead +// of silently: BACKUP_FIREWALL_RULES is always written (never "keep the stored +// value"), and EMAIL_FALLBACK_SENDMAIL is always forced true when email is on +// (never installer.ApplyInstallData's 3-branch preserve logic). +func TestCollectInstallWizardDataCLIAlwaysSendsNonNilFlags(t *testing.T) { + promptBase := config.DefaultEnvTemplate() + + t.Run("email enabled", func(t *testing.T) { + var data *installer.InstallWizardData + var err error + captureStdout(t, func() { + reader := bufio.NewReader(strings.NewReader("n\nn\nn\nn\ny\n\nn\ncron\n03:15\n")) + data, err = collectInstallWizardDataCLI(context.Background(), reader, promptBase, false, nil) + }) + if err != nil { + t.Fatalf("collectInstallWizardDataCLI error: %v", err) + } + if data.BackupFirewallRules == nil { + t.Fatal("BackupFirewallRules must never be nil") + } + if data.EmailFallbackSendmail == nil || !*data.EmailFallbackSendmail { + t.Fatalf("EmailFallbackSendmail must be non-nil true when email is on, got %v", data.EmailFallbackSendmail) + } + if data.NotificationMode != "email" { + t.Fatalf("NotificationMode = %q, want email", data.NotificationMode) + } + if strings.TrimSpace(data.EmailDeliveryMethod) == "" { + t.Fatal("EmailDeliveryMethod must never be empty when email is on") + } + if strings.TrimSpace(data.CronTime) == "" { + t.Fatal("CronTime must never be empty (it gates the SCHEDULER_TIME write)") + } + }) + + t.Run("email declined", func(t *testing.T) { + var data *installer.InstallWizardData + var err error + captureStdout(t, func() { + reader := bufio.NewReader(strings.NewReader("n\nn\ny\ny\nn\nn\ncron\n03:15\n")) + data, err = collectInstallWizardDataCLI(context.Background(), reader, promptBase, false, nil) + }) + if err != nil { + t.Fatalf("collectInstallWizardDataCLI error: %v", err) + } + if data.BackupFirewallRules == nil || !*data.BackupFirewallRules { + t.Fatalf("BackupFirewallRules must be non-nil true, got %v", data.BackupFirewallRules) + } + if data.EmailFallbackSendmail != nil { + t.Fatal("EmailFallbackSendmail must stay nil when email is off (the engine touches neither fallback key)") + } + if data.NotificationMode != "telegram" { + t.Fatalf("NotificationMode = %q, want telegram", data.NotificationMode) + } + }) +} + +// TestApplyInstallDataCLIFeedsTheEngineTheRawBase pins the OTHER half of the +// raw/expanded split: prepareBaseTemplate producing the two views is useless if the +// wizard forwards the wrong one. This cannot be covered by the characterization +// goldens - handing installer.ApplyInstallData the expanded base instead of the raw +// one is byte-identical on every golden scenario (measured), because the embedded +// template already carries BOT_TELEGRAM_TYPE=centralized and the collector always +// supplies a non-empty EmailDeliveryMethod and a non-nil EmailFallbackSendmail. So +// the Prompt view here is deliberately a base the engine would treat very +// differently, making the mistake visible. +func TestApplyInstallDataCLIFeedsTheEngineTheRawBase(t *testing.T) { + fallbackSendmail := true + firewall := false + data := &installer.InstallWizardData{ + NotificationMode: "both", + EmailDeliveryMethod: "", // blank: the engine only substitutes a stored value when editingExisting + EmailFallbackSendmail: &fallbackSendmail, + BackupFirewallRules: &firewall, + SchedulerMode: "cron", + HealthcheckMode: "off", + CronTime: "03:15", + } + + t.Run("off the edit path the engine gets the raw empty base", func(t *testing.T) { + base := installWizardBase{ + Raw: "", + Prompt: "BOT_TELEGRAM_TYPE=personal\nEMAIL_DELIVERY_METHOD=pmf\n", + } + got, err := applyInstallDataCLI(base, false, data) + if err != nil { + t.Fatalf("applyInstallDataCLI error: %v", err) + } + want, err := installer.ApplyInstallData(base.Raw, data) + if err != nil { + t.Fatalf("ApplyInstallData(raw) error: %v", err) + } + wrong, err := installer.ApplyInstallData(base.Prompt, data) + if err != nil { + t.Fatalf("ApplyInstallData(prompt) error: %v", err) + } + if want == wrong { + t.Fatal("test is vacuous: the two bases must produce different output") + } + if got != want { + t.Fatalf("the engine must receive base.Raw, not base.Prompt (got %d bytes, want %d)", len(got), len(want)) + } + prefill := installer.DeriveInstallWizardPrefill(got) + if prefill.TelegramType != "centralized" || prefill.EmailDeliveryMethod != "relay" { + t.Fatalf("a fresh install must not inherit the prompt base's stored values: telegramType=%q method=%q", + prefill.TelegramType, prefill.EmailDeliveryMethod) + } + }) + + t.Run("on the edit path the raw base is forwarded verbatim", func(t *testing.T) { + const stored = "BOT_TELEGRAM_TYPE=personal\nEMAIL_DELIVERY_METHOD=pmf\n" + base := installWizardBase{Raw: stored, Prompt: stored} + got, err := applyInstallDataCLI(base, true, data) + if err != nil { + t.Fatalf("applyInstallDataCLI error: %v", err) + } + want, err := installer.ApplyInstallData(stored, data) + if err != nil { + t.Fatalf("ApplyInstallData error: %v", err) + } + if got != want { + t.Fatalf("edit must forward the raw base untouched:\ngot:\n%s\nwant:\n%s", got, want) + } + prefill := installer.DeriveInstallWizardPrefill(got) + if prefill.TelegramType != "personal" || prefill.EmailDeliveryMethod != "pmf" { + t.Fatalf("an edit must preserve the stored values: telegramType=%q method=%q", + prefill.TelegramType, prefill.EmailDeliveryMethod) + } + }) +} + +// TestRunConfigWizardCLIRuntimeOnlyEditKeepsByteIdentity covers the second way an +// Edit base can be "blank" as far as the engine is concerned: it is non-empty on +// disk, but consists solely of runtime-derived keys that ApplyInstallData strips +// (BASE_DIR, CRON_SCHEDULE, CRON_HOUR, CRON_MINUTE). Testing only TrimSpace of the +// raw base missed it, and the appended keys landed after the leftover newline. +// +// The expectations are PARITY WITH THE PRE-REFACTOR CLI, measured at aedbe0c on a +// worktree, not what looks tidy: a base WITHOUT a trailing newline wrote 290 bytes +// and no leading blank line, while a base WITH one wrote 291 bytes AND a leading +// blank line, because the empty trailing element survives the key removal. The +// refactor must reproduce both, including the ugly one. +func TestRunConfigWizardCLIRuntimeOnlyEditKeepsByteIdentity(t *testing.T) { + for _, tc := range []struct { + name string + content string + wantLeadingBlank bool + }{ + {name: "base dir without trailing newline", content: "BASE_DIR=/opt/proxsave"}, + {name: "cron keys without trailing newline", content: "CRON_HOUR=2\nCRON_MINUTE=30"}, + {name: "export form without trailing newline", content: "export BASE_DIR=/opt"}, + { + name: "all runtime keys with trailing newline", + content: "BASE_DIR=/opt\nCRON_SCHEDULE=0 2 * * *\nCRON_HOUR=2\nCRON_MINUTE=30\n", + wantLeadingBlank: true, + }, + } { + t.Run(tc.name, func(t *testing.T) { + stubCrontabDerivation(t) + cfgFile := createTempFile(t, tc.content) + var err error + captureStdout(t, func() { + reader := bufio.NewReader(strings.NewReader("2\nn\nn\nn\nn\nn\nn\n\n03:15\n")) + _, err = runConfigWizardCLI(context.Background(), reader, cfgFile, cfgFile+".tmp", "/opt/proxsave", nil) + }) + if err != nil { + t.Fatalf("runConfigWizardCLI error: %v", err) + } + written, readErr := os.ReadFile(cfgFile) + if readErr != nil { + t.Fatalf("read back: %v", readErr) + } + if got := strings.HasPrefix(string(written), "\n"); got != tc.wantLeadingBlank { + t.Fatalf("leading blank line = %v, want %v (%d bytes):\n%q", got, tc.wantLeadingBlank, len(written), string(written)) + } + if strings.Contains(string(written), "BACKUP_ENABLED") { + t.Fatalf("runtime-only Edit must not expand to the embedded template (%d bytes)", len(written)) + } + if strings.Contains(string(written), wizardBlankEditBaseMarker) { + t.Fatalf("the blank-base marker must never reach backup.env:\n%s", written) + } + }) + } +} + +// TestPromptSanitizedNonEmptySanitizesTheDefault pins the second half of the +// sanitize guard: a stored value made of control characters reaches the prompt +// through DeriveInstallWizardPrefill unchanged, and strings.TrimSpace does not +// consider NUL to be whitespace, so it used to be OFFERED as an acceptable default +// that the sanitize check then rejected on every iteration. +// +// This does NOT restore the pre-refactor outcome -- once a non-empty-after-sanitize +// value is required, Enter-only input on a poisoned default cannot succeed either +// way. What it removes is a default the operator is invited to accept and that can +// never be accepted, which is why the assertion is on the prompt text. +func TestPromptSanitizedNonEmptySanitizesTheDefault(t *testing.T) { + var got string + var err error + out := captureStdout(t, func() { + reader := bufio.NewReader(strings.NewReader("myremote:backups\n")) + got, err = promptSanitizedNonEmptyWithDefault(context.Background(), reader, "Remote: ", "\x00\x00") + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != "myremote:backups" { + t.Fatalf("value = %q, want the typed answer", got) + } + if strings.Contains(out, "[") { + t.Fatalf("a default that can never satisfy the sanitize check must not be offered; prompt was %q", out) + } + if strings.Contains(out, "\x00") { + t.Fatalf("control characters must not be echoed into the prompt; prompt was %q", out) + } +} diff --git a/cmd/proxsave/scheduler_time_seed_test.go b/cmd/proxsave/scheduler_time_seed_test.go index 34301c93..e0c11c90 100644 --- a/cmd/proxsave/scheduler_time_seed_test.go +++ b/cmd/proxsave/scheduler_time_seed_test.go @@ -283,10 +283,10 @@ func TestPrepareBaseTemplateNeverWritesDuringTheInteractivePhase(t *testing.T) { cfg := createTempFile(t, preThirty) stubCrontabLines(t, []string{"0 21 * * * /usr/local/bin/proxsave --backup"}, nil) - var tmpl string + var base installWizardBase var err error captureStdout(t, func() { - tmpl, _, _, err = prepareBaseTemplate(context.Background(), bufio.NewReader(strings.NewReader(tt.answer)), cfg, nil) + base, _, _, err = prepareBaseTemplate(context.Background(), bufio.NewReader(strings.NewReader(tt.answer)), cfg, nil) }) if tt.wantAbort { if !errors.Is(err, errInteractiveAborted) { @@ -304,8 +304,8 @@ func TestPrepareBaseTemplateNeverWritesDuringTheInteractivePhase(t *testing.T) { t.Fatalf("the interactive phase must not write backup.env:\n%s", data) } - if got := cronTimeDefault(true, tmpl) == "21:00"; got != tt.wantPrefill { - t.Fatalf("template prefill = %v, want %v (tmpl=%q)", got, tt.wantPrefill, tmpl) + if got := cronTimeDefault(true, base.Prompt) == "21:00"; got != tt.wantPrefill { + t.Fatalf("template prefill = %v, want %v (tmpl=%q)", got, tt.wantPrefill, base.Prompt) } }) } From 5b7bb30ea4ee247dccbabe30d69be2a8a3dd90ee Mon Sep 17 00:00:00 2001 From: tis24dev Date: Mon, 3 Aug 2026 02:56:51 +0200 Subject: [PATCH 09/50] fix(support): demand the same consent on both front-ends before emailing the log Support mode attaches the operator's FULL debug log to an email addressed to the maintainer. The stdin flow demanded two explicit affirmative answers, each defaulting to No: permission to share a log that may carry personal data, and a statement that the GitHub issue already exists. The dashboard demanded neither. It showed a passive note above a metadata form, and pressing Continue -- on a screen whose visible purpose was typing a nickname and an issue number -- armed the run. Typing "#1234" is not a claim that the issue exists, and nothing checks it. Same binary, same log, same recipient, materially weaker consent depending on which front-end the operator happened to use. The disclosure and the gates now live in internal/support/consent.go, and both front-ends render them: the stdin flow asks one [y/N] per gate, the dashboard gives each gate a toggle row that blocks Continue until it reads Yes. A gate added there reaches both, which is what stops them drifting apart again. Three things the adversarial pass caught, fixed here rather than shipped: - The gate rows sit AFTER the text fields, not before them as the stdin order suggests. The grid focuses its first row on entry and a focused toggle answers to a bare "y", so with the gates first an operator whose nickname begins with y granted consent with their first keystroke -- while the stdin gate rejects that same input and re-prompts. - The shared disclosure is strictly ADDITIVE over what the two front-ends carried: the stdin flow's two lines are kept verbatim, the dashboard's contributions become warnings of their own. Sharing copy does not license rewriting it. - FormGrid reserves one field row against the fixed note. Six note lines on a short terminal used to leave Continue/Cancel alone on screen, which is a dead end once a row gates submit: refused forever by a row nobody can reach, with the refusal message dropped from the footer at those same heights. The note now truncates first and says so; below terminal height 13 it is reduced to that indicator and nothing can be acknowledged at all, which is the safe way to fail. ValidateBool is opt-in and nil everywhere else, so the installer's six toggles are untouched. Every new test is mutation-verified. --- cmd/proxsave/dashboard_support.go | 64 ++++- .../dashboard_support_consent_test.go | 213 +++++++++++++++++ internal/support/consent.go | 135 +++++++++++ internal/support/consent_test.go | 221 ++++++++++++++++++ internal/support/support.go | 64 ++--- internal/ui/components/formgrid.go | 62 ++++- internal/ui/components/formgrid_gate_test.go | 123 ++++++++++ 7 files changed, 831 insertions(+), 51 deletions(-) create mode 100644 cmd/proxsave/dashboard_support_consent_test.go create mode 100644 internal/support/consent.go create mode 100644 internal/support/consent_test.go create mode 100644 internal/ui/components/formgrid_gate_test.go diff --git a/cmd/proxsave/dashboard_support.go b/cmd/proxsave/dashboard_support.go index 365285f8..461a750d 100644 --- a/cmd/proxsave/dashboard_support.go +++ b/cmd/proxsave/dashboard_support.go @@ -16,16 +16,29 @@ import ( var dashboardRunSupportForm = runDashboardSupportForm // runDashboardSupportForm shows the SAME single-screen grid form as the installer's -// configuration screen (components.FormGrid). A consent note sits ABOVE the two fields -// (always visible, one line each): the backup runs in debug mode and its log is emailed to -// the maintainer, and the log may contain personal data such as this server's MAC. -// Below it are the GitHub nickname and the GitHub issue (#1234), each with a concise -// focused hint, plus the shared Continue / Cancel buttons. It returns (meta, true) only on -// Continue; esc / Cancel returns (_, false) so the caller loops back to the menu. The -// maintainer email address is never shown. +// configuration screen (components.FormGrid), carrying the SAME consent as the stdin +// flow (support.RunIntro): the shared disclosure and gate detail sit above the fields +// as an always-visible note, and each shared gate gets its own toggle row that must be +// set to Yes before Continue is accepted. The GitHub nickname and the GitHub issue +// (#1234) come first, each with a concise focused hint, then the gate rows, then the +// shared Continue / Cancel buttons. +// +// The gate rows sit AFTER the text fields even though the stdin flow asks them first, +// because the grid focuses its first row on entry and a focused toggle absorbs a bare +// "y" (formgrid.go, FieldToggle key handling). With the gates first, an operator whose +// GitHub nickname begins with y -- typing being the natural first act on a form that +// asks for a nickname -- silently granted consent, while the stdin gate rejects that +// same input and re-prompts. Order here is presentation only: submit() validates every +// row whatever the order, so neither acknowledgement can be skipped. +// +// It returns (meta, true) only on Continue with both acknowledgements given; esc / +// Cancel returns (_, false) so the caller loops back to the menu. The maintainer email +// address is never shown. func runDashboardSupportForm(ctx context.Context, session *shell.Session) (support.Meta, bool) { errBack := errors.New("support: back") + gates := support.ConsentGates() + nickname := &components.FormField{ Label: "GitHub nickname", Description: "Your GitHub nickname for the support request.", @@ -43,14 +56,19 @@ func runDashboardSupportForm(ctx context.Context, session *shell.Session) (suppo Kind: components.FieldText, Validate: validateSupportIssue, } + fields := append([]*components.FormField{nickname, issue}, supportConsentFields(gates)...) + + // The note is the shared disclosure plus every gate's supporting lines, so the + // operator reads the same words the stdin flow prints before its first prompt — + // here they stay on screen while the acknowledgements are given. + note := support.ConsentDisclosure.Lines() + for _, gate := range gates { + note = append(note, gate.Detail...) + } - fields := []*components.FormField{nickname, issue} if _, err := shell.Ask(ctx, session, components.NewFormGrid( "Support", fields, - components.WithFormGridNote( - "Backup run in debug mode, log will be emailed to the maintainer.", - "The log may contain personal data such as this server's MAC address.", - ), + components.WithFormGridNote(note...), components.WithFormGridBack(errBack), )); err != nil { return support.Meta{}, false // esc / Cancel / abort @@ -61,6 +79,28 @@ func runDashboardSupportForm(ctx context.Context, session *shell.Session) (suppo }, true } +// supportConsentFields turns the shared consent gates into toggle rows. Bool is left at +// its zero value: an untouched row reads "No", mirroring the stdin prompt's [y/N] +// default. Require blocks submit while the row is still No, which is what stops the +// dashboard from arming support mode on a form whose visible purpose is entering +// metadata. +// +// The default alone does not make consent deliberate -- a focused toggle also answers to +// a bare "y", so where these rows sit in the field order decides whether stray typing can +// reach them. See the ordering note on runDashboardSupportForm. +func supportConsentFields(gates []support.ConsentGate) []*components.FormField { + fields := make([]*components.FormField, 0, len(gates)) + for _, gate := range gates { + fields = append(fields, &components.FormField{ + Label: gate.Ack, + Description: gate.Question, + Kind: components.FieldToggle, + ValidateBool: gate.Require, + }) + } + return fields +} + // validateSupportIssue enforces the # issue format via the shared helper // (mirrors support.RunIntro). func validateSupportIssue(v string) error { diff --git a/cmd/proxsave/dashboard_support_consent_test.go b/cmd/proxsave/dashboard_support_consent_test.go new file mode 100644 index 00000000..55fe25ee --- /dev/null +++ b/cmd/proxsave/dashboard_support_consent_test.go @@ -0,0 +1,213 @@ +package main + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/charmbracelet/x/ansi" + + "github.com/tis24dev/proxsave/internal/support" + "github.com/tis24dev/proxsave/internal/ui/components" + "github.com/tis24dev/proxsave/internal/ui/shell" + "github.com/tis24dev/proxsave/internal/uitest" +) + +// supportFormRun drives a real runDashboardSupportForm over an observed session, the +// way TestSupportFormIsOneScreen does, and exposes what the form returned so a test can +// assert BOTH that it resolved and that it did not. +type supportFormRun struct { + driver *newkeyUIDriver + meta chan support.Meta + done chan struct{} +} + +func startSupportForm(t *testing.T) *supportFormRun { + t.Helper() + driver := &newkeyUIDriver{t: t, buf: &shell.SyncBuffer{}, pushes: make(chan string, 8)} + ctx, cancel := context.WithCancel(context.Background()) + run := &supportFormRun{driver: driver, meta: make(chan support.Meta, 1), done: make(chan struct{})} + driver.session = shell.StartObservedForTest(ctx, shell.Config{AppName: "ProxSave", Subtitle: "Dashboard"}, + driver.buf, func(title string) { driver.pushes <- title }) + go func() { + defer close(run.done) + if meta, ok := runDashboardSupportForm(ctx, driver.session); ok { + run.meta <- meta + } + }() + // Cancel and JOIN the form goroutine, so a form still waiting for input cannot + // outlive the test and keep sending into the driver. + t.Cleanup(func() { + cancel() + select { + case <-run.done: + case <-time.After(uitest.Deadline(60 * time.Second)): + t.Error("support form did not return after cancel") + } + }) + driver.waitScreen("Support") + return run +} + +// stillOpen asserts the form has NOT resolved. It is paired with a waitOutput on the +// rejection message that follows it: the wait here catches an immediate resolve, and +// the rejection render then proves submit really ran and took the reject branch (so a +// gate that stopped blocking cannot slip through as a slow submit). +func (r *supportFormRun) stillOpen(t *testing.T, why string) { + t.Helper() + select { + case <-r.done: + t.Fatalf("%s\n%s", why, tailStr(ansi.Strip(r.driver.buf.String()))) + case <-time.After(uitest.Deadline(time.Second)): + } +} + +// TestSupportFormBlocksUntilBothConsentGatesAreSet is the D1 regression test. +// +// Support mode emails the operator's FULL debug log to the maintainer. The stdin flow +// demands two explicit affirmative answers first; the dashboard used to arm the very +// same run from a form whose only visible purpose was entering a nickname and an issue +// number, so pressing Continue there was never a statement of consent. Filling both +// text fields and pressing Continue must now be REFUSED while either acknowledgement is +// still No, and refused separately for each one — the run is armed only after both. +func TestSupportFormBlocksUntilBothConsentGatesAreSet(t *testing.T) { + gates := support.ConsentGates() + if len(gates) != 2 { + t.Fatalf("this test drives the form by row index; it needs the two shared gates, got %d", len(gates)) + } + run := startSupportForm(t) + d := run.driver + + // Row order is: the two text fields, then the two gate toggles (untouched, i.e. No). + // Navigation is "down" only, so no key can flip a toggle by accident. + d.typeText("alice") // row 0, GitHub nickname, focused on entry + d.keys("down") // -> GitHub issue (row 1) + d.typeText("#123") + d.keys("down down down") // -> past both gate rows to the buttons row + d.keys("enter") // press Continue with both acknowledgements still No + + run.stillOpen(t, "the form resolved with both consent rows still No: the dashboard would arm support mode without consent") + d.waitOutput(gates[0].Unmet) + + // Give only the FIRST acknowledgement (the rejection parks the cursor on it) and + // press Continue again: the second gate must reject on its own. + d.keys("y") + d.keys("down down") // row 2 -> the buttons row + d.keys("enter") // Continue + run.stillOpen(t, "the form resolved with only the first acknowledgement given") + d.waitOutput(gates[1].Unmet) + + // Both acknowledgements: the form resolves and carries the metadata. + d.keys("y") + d.keys("down") // row 3 -> the buttons row + d.keys("enter") // Continue + + select { + case meta := <-run.meta: + if meta.GitHubUser != "alice" || meta.IssueID != "#123" { + t.Fatalf("meta = %+v; want GitHubUser=alice IssueID=#123", meta) + } + case <-time.After(uitest.Deadline(30 * time.Second)): + t.Fatalf("the form must resolve once both acknowledgements are given:\n%s", tailStr(ansi.Strip(d.buf.String()))) + } +} + +// TestSupportConsentFieldsDefaultToNo pins the safety default: the rows the dashboard +// builds from the shared gates start unchecked and refuse submit in that state, so +// consent can only ever be the result of an explicit act. A default of Yes would put +// the form back to arming support mode on a bare Continue. +func TestSupportConsentFieldsDefaultToNo(t *testing.T) { + gates := support.ConsentGates() + fields := supportConsentFields(gates) + if len(fields) != len(gates) { + t.Fatalf("every shared gate must get a row: %d rows for %d gates", len(fields), len(gates)) + } + for i, f := range fields { + gate := gates[i] + if f.Kind != components.FieldToggle { + t.Errorf("row %q must be a toggle, got kind %v", f.Label, f.Kind) + } + if f.Bool { + t.Errorf("row %q must default to No", f.Label) + } + // Label and hint come from the shared gate, so the dashboard cannot label a + // row with something the stdin flow never asks. + if f.Label != gate.Ack { + t.Errorf("row %d label = %q; want the shared %q", i, f.Label, gate.Ack) + } + if f.Description != gate.Question { + t.Errorf("row %d hint = %q; want the shared question %q", i, f.Description, gate.Question) + } + if f.ValidateBool == nil { + t.Fatalf("row %q must gate submit (ValidateBool is nil)", f.Label) + } + if err := f.ValidateBool(false); err == nil { + t.Errorf("row %q must refuse submit while it is No", f.Label) + } else if !strings.Contains(err.Error(), gate.Unmet) { + t.Errorf("row %q rejection = %q; want the shared %q", f.Label, err, gate.Unmet) + } + if err := f.ValidateBool(true); err != nil { + t.Errorf("row %q must accept submit once it is Yes, got %v", f.Label, err) + } + } +} + +// TestSupportFormRendersSharedConsentCopy is the dashboard half of the shared-copy +// check (its counterpart is TestRunIntroRendersSharedConsentCopy in internal/support). +// It walks the shared values rather than quoting them, so rewording consent.go keeps it +// green while this front-end silently dropping a line — the state D1 came from — fails +// it. +func TestSupportFormRendersSharedConsentCopy(t *testing.T) { + run := startSupportForm(t) + d := run.driver + + // The disclosure and every gate's supporting lines are the always-visible note. + for _, line := range support.ConsentDisclosure.Lines() { + d.waitOutput(line) + } + gates := support.ConsentGates() + for _, gate := range gates { + for _, detail := range gate.Detail { + d.waitOutput(detail) + } + // Every gate gets its own row, labelled with the shared affirmative. + d.waitOutput(gate.Ack) + } + // The question a gate asks is that row's hint, which renders only while the row is + // focused -- and entry focus is the nickname field, not a gate (see the ordering + // note on runDashboardSupportForm). Navigating there to assert it is not reliable + // here: only the FIRST frame of a screen is a full render, and bubbletea's cell-diff + // renderer then emits just the changed cells, which splits a re-rendered line in + // this cumulative buffer (driving "down" produced a buffer holding + // "...Yes No confirm that you have already opened a GitHub issue?", i.e. the hint + // without its "Do you " prefix). The hints are pinned at the field level by + // TestSupportConsentFieldsDefaultToNo instead. +} + +// TestSupportFormEntryFocusKeepsTypingOffTheConsentRows pins the field ORDER as a +// consent property, not a cosmetic one. +// +// A focused toggle answers to a bare "y" (formgrid.go, FieldToggle key handling). With +// the gate rows first -- which is the order the stdin flow asks in -- the row focused on +// entry was a consent toggle, so an operator typing a GitHub nickname that begins with y +// granted consent with the first keystroke and never knew: "ydev" armed the run, while +// the stdin gate rejects that same answer and re-prompts. Typing is the natural first act +// on a form asking for a nickname, which is what made it reachable. +func TestSupportFormEntryFocusKeepsTypingOffTheConsentRows(t *testing.T) { + gates := support.ConsentGates() + run := startSupportForm(t) + d := run.driver + + // Type a y-leading nickname as the very first act, exactly as an operator would. + d.typeText("ydev") + d.keys("down") + d.typeText("#123") + d.keys("down down down") // past both gate rows, touching neither + d.keys("enter") // Continue + + run.stillOpen(t, "the form resolved after only a nickname and an issue were typed: consent was granted by a keystroke meant for a text field") + // The FIRST gate must be the one that rejects. If the typed "y" had reached it, the + // rejection would be the second gate's instead, and this wait would time out. + d.waitOutput(gates[0].Unmet) +} diff --git a/internal/support/consent.go b/internal/support/consent.go new file mode 100644 index 00000000..57a08608 --- /dev/null +++ b/internal/support/consent.go @@ -0,0 +1,135 @@ +package support + +import "errors" + +// Consent copy and gates for support mode, single-sourced for EVERY front-end. +// +// Support mode attaches the operator's full debug log to an email addressed to the +// maintainer (SendEmail builds its EmailConfig with AttachLogFile: true). That is one +// privacy-relevant act, so the stdin flow (RunIntro) and the dashboard form must +// disclose the same thing and demand the same acknowledgements. They previously did +// not: the dashboard armed support mode from a metadata form with a passive note, with +// no affirmative consent and no issue-already-open question at all. Keeping the copy +// and the gates here — rather than as literals inside each renderer — is what stops +// the two from drifting apart again; a front-end that stops rendering these values is +// caught by the tests that walk ConsentDisclosure.Lines() and ConsentGates(). +// +// Renderers own presentation only (color, layout, toggle vs. y/n prompt). The values +// below carry no ANSI on purpose: the dashboard feeds them to components.FormGrid, +// which sanitizes its note lines and every field label/hint, so an escape sequence +// written here would be stripped there rather than rendered. + +// Disclosure is what the operator must be shown BEFORE being asked anything: what +// support mode does (Summary) and what it costs them in privacy terms (Warnings). +// The split exists so a renderer can emphasise the privacy lines — the CLI prints +// them in yellow — without owning the words. +type Disclosure struct { + Summary string + Warnings []string +} + +// Lines returns the disclosure in the order every front-end must show it: the +// summary first, then each warning. The slice is freshly allocated so a caller can +// append its own lines (the dashboard appends the gate detail) without writing +// through to the package value. +// +// The stdin flow does NOT render through this method -- it prints Summary and Warnings +// separately because it emphasises the warnings in yellow, which needs the split -- so +// it is held to this order by TestRunIntroFollowsDisclosureLineOrder instead. Without +// that test the two front-ends could disclose the same lines in different orders. +func (d Disclosure) Lines() []string { + lines := make([]string, 0, len(d.Warnings)+1) + if d.Summary != "" { + lines = append(lines, d.Summary) + } + return append(lines, d.Warnings...) +} + +// ConsentDisclosure is the shared disclosure, built STRICTLY ADDITIVELY over what the +// two front-ends carried on their own: the stdin flow's two lines are kept verbatim and +// the dashboard's two contributions (the maintainer as the recipient, the MAC address as +// a concrete example of what leaks) become warnings of their own. Sharing the copy does +// not license rewriting it -- a reworded summary would change CLI stdout for nothing, +// and the additive form was verified to satisfy every test the reworded one did. +// +// Line length is load-bearing: FormGrid renders the note at min(width, 100) columns and +// wraps past that, which no test can then match with a Contains. Keep each line short. +var ConsentDisclosure = Disclosure{ + Summary: "This mode will send the ProxSave log to the developer for debugging.", + Warnings: []string{ + "If your log contains personal or sensitive information, it will be shared.", + "The full log is emailed to the maintainer at the end of the run.", + "The log may contain personal data such as this server's MAC address.", + }, +} + +// ConsentGate is ONE acknowledgement the operator has to give before support mode may +// be armed. Both front-ends render the same gate values, so neither can end up asking +// for less than the other. +type ConsentGate struct { + // Question is the acknowledgement itself, phrased as a yes/no question and + // WITHOUT the "[y/N]: " suffix Prompt appends. The stdin flow asks it verbatim; + // the dashboard shows it as the hint of the row that carries the toggle. + Question string + // Ack is the short affirmative form for a renderer that labels a control rather + // than asking a question — the dashboard's toggle row. It says what a Yes means, + // so the label and the Question can never disagree about which answer consents. + Ack string + // Detail are the supporting lines that must be shown together with the question + // (they explain why the acknowledgement is being asked for). Both front-ends + // display them before/alongside the gate; may be empty when the disclosure + // already carries the reason. + Detail []string + // DeclineWarning is the exact bootstrap.Warning text emitted when the gate is + // refused on the stdin path. + DeclineWarning string + // Unmet is the message a form renderer shows inline while the acknowledgement is + // still missing. It is phrased as "what you must do", not as DeclineWarning's + // past-tense abort, because at that point nothing has been aborted: the form + // simply refuses to submit. + Unmet string +} + +// Prompt is the stdin prompt for this gate. The capital N advertises the default that +// promptYesNoSupport implements (an empty answer returns false), so silence at the +// prompt is a refusal. +func (g ConsentGate) Prompt() string { return g.Question + " [y/N]: " } + +// Require reports whether the acknowledgement was given, as an error a form validator +// can surface inline. It is the toggle-side counterpart of the stdin default-to-No: +// the zero value of a bool control is false, so an untouched control never consents. +func (g ConsentGate) Require(given bool) error { + if given { + return nil + } + return errors.New(g.Unmet) +} + +// ConsentGateAccept is the consent gate proper: permission to send the log at all. +var ConsentGateAccept = ConsentGate{ + Question: "Do you accept and continue?", + Ack: "I accept and continue", + DeclineWarning: "Support mode aborted by user (consent not granted)", + Unmet: "consent is required before the log can be emailed", +} + +// ConsentGateIssueOpen is the second gate: the operator states that the GitHub issue +// the report belongs to already exists. Typing an issue number is not that statement — +// nothing checks the number against GitHub — which is why it is asked separately. +var ConsentGateIssueOpen = ConsentGate{ + Question: "Do you confirm that you have already opened a GitHub issue?", + Ack: "Issue already open on GitHub", + Detail: []string{ + "Before proceeding, you must have an open GitHub issue for this problem.", + "Emails without a corresponding GitHub issue will not be analyzed.", + }, + DeclineWarning: "Support mode aborted: please open a GitHub issue first", + Unmet: "open the issue on GitHub first, then confirm here", +} + +// ConsentGates returns every gate, in the order they must be presented. Front-ends +// iterate this instead of naming the gates one by one, so a gate added here shows up +// in both of them. The slice is freshly allocated per call. +func ConsentGates() []ConsentGate { + return []ConsentGate{ConsentGateAccept, ConsentGateIssueOpen} +} diff --git a/internal/support/consent_test.go b/internal/support/consent_test.go new file mode 100644 index 00000000..26a925c5 --- /dev/null +++ b/internal/support/consent_test.go @@ -0,0 +1,221 @@ +package support + +import ( + "bufio" + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/tis24dev/proxsave/internal/logging" +) + +// captureStdout points os.Stdout at a temp file for the rest of the test and returns a +// reader for everything written to it. RunIntro prints through fmt.Print*, which reads +// os.Stdout at call time, so swapping the variable captures it; the file is unbuffered, +// so the reader sees the writes without closing it first. +func captureStdout(t *testing.T) func() string { + t.Helper() + path := filepath.Join(t.TempDir(), "stdout.txt") + f, err := os.Create(path) + if err != nil { + t.Fatalf("create stdout capture: %v", err) + } + orig := os.Stdout + os.Stdout = f + t.Cleanup(func() { + os.Stdout = orig + _ = f.Close() + }) + return func() string { + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read stdout capture: %v", err) + } + return string(data) + } +} + +// TestRunIntroRendersSharedConsentCopy: the stdin front-end must render the SHARED +// consent data (consent.go), not prose of its own, so it cannot drift away from the +// dashboard form — the defect this pins is exactly that the two front-ends demanded +// different consent for the same act (emailing the full debug log). +// +// The assertion follows the data instead of quoting it: rewording ConsentDisclosure or +// a gate keeps this test passing, while a front-end that stops rendering one of those +// values fails it. Its counterpart on the dashboard side is +// TestSupportFormRendersSharedConsentCopy in package main. +func TestRunIntroRendersSharedConsentCopy(t *testing.T) { + withStdinFile(t, strings.Join([]string{"y", "y", "user", "#123", ""}, "\n")) + stdout := captureStdout(t) + + _, ok, interrupted := RunIntro(context.Background(), logging.NewBootstrapLogger()) + got := stdout() + if !ok || interrupted { + t.Fatalf("ok=%v interrupted=%v; want true/false\n%s", ok, interrupted, got) + } + + for _, line := range ConsentDisclosure.Lines() { + if !strings.Contains(got, line) { + t.Errorf("the stdin flow must show the shared disclosure line %q; output:\n%s", line, got) + } + } + + // Every gate is asked, with its supporting lines, in ConsentGates() order. The + // order matters: the operator must consent to sharing the log before being asked + // anything else about it. + prev := -1 + for _, gate := range ConsentGates() { + at := strings.Index(got, gate.Prompt()) + if at < 0 { + t.Errorf("the stdin flow must ask the shared gate %q; output:\n%s", gate.Prompt(), got) + continue + } + if at <= prev { + t.Errorf("gate %q was asked out of ConsentGates() order (index %d, previous %d)", gate.Question, at, prev) + } + prev = at + for _, detail := range gate.Detail { + if !strings.Contains(got, detail) { + t.Errorf("the stdin flow must show %q with gate %q; output:\n%s", detail, gate.Question, got) + } + } + } +} + +// TestConsentDisclosureKeepsTheLoadBearingWords anchors the shared disclosure to +// LITERALS, deliberately. +// +// Every other consent assertion walks ConsentDisclosure/ConsentGates, so it moves with +// the data: making Lines() return nothing keeps BOTH "renders the shared copy" tests +// green while the dashboard renders two consent toggles above an empty note -- an +// operator asked to accept, with nothing on screen saying what. These are the four facts +// they are accepting, and losing any of them is what this catches. +func TestConsentDisclosureKeepsTheLoadBearingWords(t *testing.T) { + lines := ConsentDisclosure.Lines() + joined := strings.Join(lines, "\n") + for _, want := range []string{ + "send the ProxSave log", // what leaves the host + "it will be shared", // that someone else receives it + "emailed to the maintainer", // who that someone is + "MAC address", // a concrete example of what it can carry + } { + if !strings.Contains(joined, want) { + t.Errorf("the shared disclosure must still state %q; got:\n%s", want, joined) + } + } + // The dashboard renders these lines through components.FormGrid, which lays the note + // out at min(width, 100) columns and wraps past that. A wrapped line is no longer + // findable as one string, so the dashboard-side copy test stops matching it and fails + // only by timing out, 60s later, with a message that never mentions width. + for _, line := range lines { + if len(line) > 90 { + t.Errorf("disclosure line is %d chars, too long to render unwrapped: %q", len(line), line) + } + } +} + +// TestConsentGatePromptAdvertisesTheDefault: the capital N is the only thing telling the +// operator that silence at the prompt refuses, and the trailing space keeps their answer +// on the prompt's own line. Both are invisible to every data-driven assertion here -- +// they all search for Prompt(), so they follow it wherever it goes. +func TestConsentGatePromptAdvertisesTheDefault(t *testing.T) { + for _, gate := range ConsentGates() { + if got := gate.Prompt(); got != gate.Question+" [y/N]: " { + t.Errorf("gate prompt = %q; want the question followed by %q", got, " [y/N]: ") + } + } +} + +// TestRunIntroFollowsDisclosureLineOrder holds the stdin flow to Lines() order without +// making it render through Lines(): it prints Summary and Warnings separately because it +// emphasises the warnings in yellow. Nothing else pins that, so the two front-ends could +// disclose the same lines in different orders with a green suite. +// +// It also pins that each gate's supporting lines are shown BEFORE that gate is asked. +// Moving them after the answer would have the operator confirm they opened a GitHub +// issue before being told why it is required. +func TestRunIntroFollowsDisclosureLineOrder(t *testing.T) { + withStdinFile(t, strings.Join([]string{"y", "y", "user", "#123", ""}, "\n")) + stdout := captureStdout(t) + + _, ok, interrupted := RunIntro(context.Background(), logging.NewBootstrapLogger()) + got := stdout() + if !ok || interrupted { + t.Fatalf("ok=%v interrupted=%v; want true/false\n%s", ok, interrupted, got) + } + + prev := -1 + for _, line := range ConsentDisclosure.Lines() { + at := strings.Index(got, line) + if at < 0 { + t.Fatalf("the stdin flow must show %q; output:\n%s", line, got) + } + if at <= prev { + t.Errorf("disclosure line %q is out of Lines() order (index %d, previous %d)", line, at, prev) + } + prev = at + } + + for _, gate := range ConsentGates() { + asked := strings.Index(got, gate.Prompt()) + if asked < 0 { + t.Fatalf("the stdin flow must ask %q; output:\n%s", gate.Prompt(), got) + } + for _, detail := range gate.Detail { + at := strings.Index(got, detail) + if at < 0 || at > asked { + t.Errorf("%q must be shown BEFORE the gate asks %q (detail at %d, prompt at %d)", detail, gate.Question, at, asked) + } + } + } +} + +// TestPromptYesNoSupportEmptyAnswerIsNo pins the [y/N] default that the whole consent +// gate rests on: an operator who just presses Enter has NOT agreed to email their debug +// log. If an empty answer returned true, both gates would pass on silence. +func TestPromptYesNoSupportEmptyAnswerIsNo(t *testing.T) { + for _, answer := range []string{"\n", " \n"} { + granted, err := promptYesNoSupport(context.Background(), bufio.NewReader(strings.NewReader(answer)), "prompt: ") + if err != nil { + t.Fatalf("promptYesNoSupport(%q) error: %v", answer, err) + } + if granted { + t.Errorf("an empty answer (%q) must default to No, got granted=true", answer) + } + } +} + +// TestConsentGatesOrder pins the sequence itself, which the data-driven assertions +// above deliberately cannot: they walk ConsentGates(), so reordering that slice moves +// the production render and the expectation together. Consent to sharing the log has +// to be asked FIRST — an operator must not be asked to state anything about a report +// they have not yet agreed to send. +func TestConsentGatesOrder(t *testing.T) { + gates := ConsentGates() + if len(gates) != 2 { + t.Fatalf("expected the two consent gates, got %d", len(gates)) + } + if gates[0].Question != ConsentGateAccept.Question { + t.Errorf("the consent gate must come first, got %q", gates[0].Question) + } + if gates[1].Question != ConsentGateIssueOpen.Question { + t.Errorf("the issue-already-open gate must come second, got %q", gates[1].Question) + } +} + +// TestConsentGateRequire: Require is the form-renderer side of that same default — +// a control left at its zero value (false) must produce a blocking error. +func TestConsentGateRequire(t *testing.T) { + for _, gate := range ConsentGates() { + if err := gate.Require(false); err == nil { + t.Errorf("gate %q must reject a missing acknowledgement", gate.Question) + } else if !strings.Contains(err.Error(), gate.Unmet) { + t.Errorf("gate %q must explain what is missing, got %q; want %q", gate.Question, err, gate.Unmet) + } + if err := gate.Require(true); err != nil { + t.Errorf("gate %q must accept an explicit acknowledgement, got %v", gate.Question, err) + } + } +} diff --git a/internal/support/support.go b/internal/support/support.go index 5f782267..852d09f5 100644 --- a/internal/support/support.go +++ b/internal/support/support.go @@ -65,6 +65,13 @@ func ValidateIssueID(v string) error { // RunIntro prompts for consent and GitHub metadata. // ok=false means the user declined or aborted; interrupted=true means context cancel / Ctrl+C. +// +// The disclosure and the gates are NOT written here: they come from ConsentDisclosure +// and ConsentGates() so this flow and the dashboard form ask for exactly the same +// consent (see consent.go). This function only renders them for a terminal and owns +// the stdin protocol: ONE bufio.Reader for the whole flow (a second one would drop the +// bytes the first already buffered and orphan input.ReadLineWithIdle's per-reader +// state), then one yes/no read per gate, then the nickname and issue loops. func RunIntro(ctx context.Context, bootstrap *logging.BootstrapLogger) (meta Meta, ok bool, interrupted bool) { reader := bufio.NewReader(os.Stdin) @@ -73,41 +80,36 @@ func RunIntro(ctx context.Context, bootstrap *logging.BootstrapLogger) (meta Met fmt.Println("\033[32m SUPPORT & ASSISTANCE MODE\033[0m") fmt.Println("\033[32m================================================\033[0m") fmt.Println() - fmt.Println("This mode will send the ProxSave log to the developer for debugging.") - fmt.Println("\033[33mIf your log contains personal or sensitive information, it will be shared.\033[0m") - fmt.Println() - - accepted, err := promptYesNoSupport(ctx, reader, "Do you accept and continue? [y/N]: ") - if err != nil { - if errors.Is(err, input.ErrInputAborted) || ctx.Err() == context.Canceled { - bootstrap.Warning("Support mode interrupted by signal") - return Meta{}, false, true - } - bootstrap.Error("ERROR: %v", err) - return Meta{}, false, false - } - if !accepted { - bootstrap.Warning("Support mode aborted by user (consent not granted)") - return Meta{}, false, false + fmt.Println(ConsentDisclosure.Summary) + for _, warning := range ConsentDisclosure.Warnings { + // Yellow is this renderer's emphasis, not part of the shared copy. + fmt.Printf("\033[33m%s\033[0m\n", warning) } - - fmt.Println() - fmt.Println("Before proceeding, you must have an open GitHub issue for this problem.") - fmt.Println("Emails without a corresponding GitHub issue will not be analyzed.") fmt.Println() - hasIssue, err := promptYesNoSupport(ctx, reader, "Do you confirm that you have already opened a GitHub issue? [y/N]: ") - if err != nil { - if errors.Is(err, input.ErrInputAborted) || ctx.Err() == context.Canceled { - bootstrap.Warning("Support mode interrupted by signal") - return Meta{}, false, true + for _, gate := range ConsentGates() { + if len(gate.Detail) > 0 { + // Blank above and below so the supporting lines read as their own block + // instead of trailing off the previous prompt line. + fmt.Println() + for _, line := range gate.Detail { + fmt.Println(line) + } + fmt.Println() + } + granted, err := promptYesNoSupport(ctx, reader, gate.Prompt()) + if err != nil { + if errors.Is(err, input.ErrInputAborted) || ctx.Err() == context.Canceled { + bootstrap.Warning("Support mode interrupted by signal") + return Meta{}, false, true + } + bootstrap.Error("ERROR: %v", err) + return Meta{}, false, false + } + if !granted { + bootstrap.Warning("%s", gate.DeclineWarning) + return Meta{}, false, false } - bootstrap.Error("ERROR: %v", err) - return Meta{}, false, false - } - if !hasIssue { - bootstrap.Warning("Support mode aborted: please open a GitHub issue first") - return Meta{}, false, false } // GitHub nickname diff --git a/internal/ui/components/formgrid.go b/internal/ui/components/formgrid.go index f54e667f..5e9c7aa7 100644 --- a/internal/ui/components/formgrid.go +++ b/internal/ui/components/formgrid.go @@ -40,6 +40,12 @@ type FormField struct { // Validate rejects a text value on submit (and inline when leaving the // field); only called while the field is Active. Validate func(value string) error + // ValidateBool rejects a toggle value on submit; only called while the field + // is Active and Kind is FieldToggle. It is what lets a toggle be a GATE (an + // acknowledgement the operator must give before the form may resolve) rather + // than a setting. nil — the zero value, so every field that does not opt in — + // means the toggle never blocks submit. + ValidateBool func(value bool) error // Active gates the field: inactive rows render dimmed, are skipped by // navigation, and are not validated. nil = always active. Active func() bool @@ -196,14 +202,37 @@ func (g *FormGrid) Help() string { return "↑/↓ move · ←/→/space toggle · enter next/confirm · esc cancel" } +// submit is the ONLY place the grid resolves successfully; every other Resolve call +// carries backErr. Both ways to press Continue — Enter on the buttons row and a left +// click on the Continue band — return through here, so a field that rejects submit +// blocks the mouse exactly as it blocks the keyboard. Validating in the Enter handler +// instead would leave the click path (which force-sets the cursor and calls submit +// directly) ungated. func (g *FormGrid) submit() (shell.Screen, tea.Cmd) { - // Sync the editor, then validate every active text field in order. + // Sync the editor, then validate every active field in order. g.bindEditor() for i, f := range g.fields { - if !f.active() || f.Kind != FieldText || f.Validate == nil { + if !f.active() { continue } - if err := f.Validate(f.Text); err != nil { + var err error + switch f.Kind { + case FieldText: + if f.Validate == nil { + continue + } + err = f.Validate(f.Text) + case FieldToggle: + if f.ValidateBool == nil { + continue + } + err = f.ValidateBool(f.Bool) + default: + continue + } + if err != nil { + // Same rejection shape for both kinds: label-prefixed inline message and + // the cursor parked on the control the operator has to change. g.errMsg = fmt.Sprintf("%s: %v", f.Label, err) g.cursor = i return g, g.bindEditor() @@ -492,7 +521,18 @@ func (g *FormGrid) View(width, height int) string { // last kept line with a subtle truncation indicator so the crop is visible. // Recompute introHeight from the clipped intro so head, the builder and // g.lastRowsTop all derive from the same lines (hit-testing stays consistent). - if maxIntro := max(height-5, 0); len(intro) > 0 && introHeight > maxIntro { + // Reserve one field row when the grid has any. The note is FIXED -- it never + // scrolls -- so on a short terminal it can absorb the whole budget and leave nothing + // but Continue/Cancel on screen. That is a dead end as soon as a row gates submit + // (the support consent toggles): Continue is refused by a row the operator has no way + // to reach, and at those heights the refusal message is dropped too, so pressing it + // does nothing visible. Reserving the row makes the note truncate first, which it + // already announces. + minRows := 0 + if len(g.fields) > 0 { + minRows = 1 + } + if maxIntro := max(height-5-minRows, 0); len(intro) > 0 && introHeight > maxIntro { kept := make([]string, 0, len(intro)) used := 0 dropped := false @@ -509,7 +549,11 @@ func (g *FormGrid) View(width, height int) string { indicator := theme.Subtle.Width(introWidth).Render("note truncated, enlarge the terminal") if len(kept) > 0 { kept[len(kept)-1] = indicator - } else if maxIntro >= 1 { + } else { + // Emitted even when no room was reserved for it, at the cost of the + // field row above: a note that vanishes silently would leave the + // consent toggles on screen with nothing left stating what is being + // acknowledged. Refusing to show either is the safe way to fail. kept = append(kept, indicator) } } @@ -528,14 +572,16 @@ func (g *FormGrid) View(width, height int) string { // Reserve the buttons block (blank + buttons = 2 rows) as the TOP priority so // the actionable rows are never cropped from below (the router crops overflow // from the bottom, like Confirm's budget). The footer (hint/error) is lower - // priority: drop it when there is no room, and let the field window shrink - // into the remainder (down to zero at extreme sizes, recoverable by enlarging). + // priority: drop it when keeping it would starve the field window below the row + // reserved above, so an actionable row always outranks the hint. The window can + // still reach zero at the extreme size where the note is reduced to its truncation + // indicator alone -- recoverable by enlarging, which is what the indicator says. buttonsLines := 2 // blank + buttons footerBlock := footerHeight if len(footer) > 0 { footerBlock++ // blank line between the buttons and the footer } - if height-head-buttonsLines-footerBlock < 0 { + if height-head-buttonsLines-footerBlock < minRows { footer = nil footerBlock = 0 } diff --git a/internal/ui/components/formgrid_gate_test.go b/internal/ui/components/formgrid_gate_test.go new file mode 100644 index 00000000..24c225f1 --- /dev/null +++ b/internal/ui/components/formgrid_gate_test.go @@ -0,0 +1,123 @@ +package components + +import ( + "errors" + "strings" + "testing" +) + +// gateGrid builds a form with a gating toggle in front of a valid text field, so the +// only thing that can block submit is the toggle. +func gateGrid() (*FormGrid, *FormField) { + gate := &FormField{ + Label: "I accept", + Kind: FieldToggle, + ValidateBool: func(v bool) error { + if !v { + return errors.New("acknowledgement required") + } + return nil + }, + } + nick := &FormField{Label: "Nick", Kind: FieldText, Text: "alice"} + return NewFormGrid("Consent", []*FormField{gate, nick}), gate +} + +// TestFormGridToggleGateBlocksSubmit: a toggle whose ValidateBool rejects the current +// value must block Continue on BOTH ways of pressing it — Enter on the buttons row and +// a left click on the Continue band — with the same label-prefixed inline message and +// cursor jump the text validators produce. The click path force-sets the cursor and +// calls submit directly, 70 lines away from the key handler, so it is the one a gate +// added in the wrong place would miss. +func TestFormGridToggleGateBlocksSubmit(t *testing.T) { + g, gate := gateGrid() + cap := bindGrid(g) + + press(t, g, "down") // gate -> Nick + press(t, g, "down") // Nick -> buttons (Continue focused) + press(t, g, "enter") // Continue + if cap.resolved { + t.Fatal("an unmet toggle gate must block submit") + } + view := g.View(100, 20) + if !strings.Contains(view, "I accept: acknowledgement required") { + t.Fatalf("the gate message must be shown inline, prefixed with the row label:\n%s", view) + } + if g.cursor != 0 { + t.Fatalf("the cursor must park on the row that has to change, cursor=%d", g.cursor) + } + + // Same grid, same unmet gate, pressed with the mouse. + g.Update(click(g.contX0+1, g.lastButtonsY)) //nolint:errcheck + if cap.resolved { + t.Fatal("an unmet toggle gate must block a Continue CLICK too") + } + + // The acknowledgement given (the cursor is parked on the gate row), the same + // Continue now resolves. + press(t, g, "y") + if !gate.Bool { + t.Fatalf("test setup: y must set the gate row to Yes") + } + press(t, g, "down") + press(t, g, "down") + press(t, g, "enter") + if !cap.resolved || cap.err != nil { + t.Fatalf("submit must be accepted once the gate is satisfied, got %+v", cap) + } +} + +// TestFormGridTallNoteNeverHidesEveryFieldRow: the note is fixed — it never scrolls — so +// on a short terminal it used to absorb the whole height budget and render a form with +// NO field rows at all, just Continue/Cancel. On a settings form that is merely ugly; on +// a GATING form it is a dead end, because submit is then refused by a row the operator +// cannot reach, and at those same heights the refusal message is dropped from the footer +// too, so pressing Continue does nothing visible. The note must truncate first — which it +// announces — and it must never disappear in silence either: a vanished consent note +// would leave the acknowledgement rows on screen with nothing stating what is accepted. +func TestFormGridTallNoteNeverHidesEveryFieldRow(t *testing.T) { + note := []string{"note one", "note two", "note three", "note four", "note five", "note six"} + const truncated = "note truncated" + + for height := 7; height <= 16; height++ { + g, _ := gateGrid() + g = NewFormGrid("Consent", g.fields, WithFormGridNote(note...)) + view := g.View(100, height) + if !strings.Contains(view, "I accept") && !strings.Contains(view, "Nick") { + t.Errorf("height %d renders no field row, so a gating form cannot be completed:\n%s", height, view) + } + if strings.Contains(view, note[len(note)-1]) { + continue // the whole note fits, nothing to announce + } + if !strings.Contains(view, truncated) { + t.Errorf("height %d drops note lines without saying so:\n%s", height, view) + } + } + + // Below that the note is reduced to the indicator alone and the field rows go with + // it. That is the safe way to fail: nothing can be acknowledged, and the line on + // screen says what to do about it. + for height := 3; height <= 6; height++ { + g, _ := gateGrid() + g = NewFormGrid("Consent", g.fields, WithFormGridNote(note...)) + if view := g.View(100, height); !strings.Contains(view, truncated) { + t.Errorf("height %d must still say the note was truncated:\n%s", height, view) + } + } +} + +// TestFormGridToggleWithoutValidateBoolSubmits: ValidateBool is opt-in. A plain toggle +// (the installer's six settings rows) leaves it nil and must submit at either value — +// adding the hook may not turn existing settings into gates. +func TestFormGridToggleWithoutValidateBoolSubmits(t *testing.T) { + for _, on := range []bool{false, true} { + toggle := &FormField{Label: "Cloud backups (rclone)", Kind: FieldToggle, Bool: on} + g := NewFormGrid("Configuration", []*FormField{toggle}) + cap := bindGrid(g) + press(t, g, "down") // toggle -> buttons + press(t, g, "enter") // Continue + if !cap.resolved || cap.err != nil { + t.Fatalf("a toggle with a nil ValidateBool must not block submit (Bool=%v), got %+v", on, cap) + } + } +} From 52f74d098b2dcdc6a8c8badf56a5657ae9f51b7c Mon Sep 17 00:00:00 2001 From: tis24dev Date: Mon, 3 Aug 2026 03:29:24 +0200 Subject: [PATCH 10/50] fix(install): stop leaking raw relay bytes into the TUI install log, and record why the healthcheck step was skipped MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two real defects on the graphical install path, found while comparing it with the stdin path. The retry loops the two share were deliberately NOT unified: their navigation differs enough that the abstraction would cost more than the duplicate. 1. The relay's status message reached the persisted install log unscrubbed. installer.TelegramSetupResult.LastStatusMessage held whatever the Telegram relay sent, raw and by design ("parity with tview"), and the TUI driver wrote it into the bootstrap log. That log is written to disk and read back with cat/less, so a hostile relay response could move the cursor of whoever debugs the install. The CLI had scrubbed the same string from the start. Fixed at the WRITE site, not the log site: the field exists to be logged and carries untrusted input, so scrubbing where it is assigned is what protects every future consumer instead of trusting each one to remember. The log line then routes through the new orchestrator.TelegramSetupStatusMessageForLog, which also supplies the stand-in for a relay that said nothing — the CLI's own inline fallback now calls the same helper, so the two install logs cannot drift. 2. The healthcheck step's skip diagnosis never reached a TUI install log. logHealthcheckSetupBootstrapOutcome was called by the CLI and by nothing else, so every reason the step can decline to run (no alive URL yet, unreadable config, no server identity, self mode) was simply absent. The step read as if it had never happened and the reason was unrecoverable afterwards. Both log trails move out of the driver into logTelegramSetupOutcome / logHealthcheckSetupOutcome, one shape each. They were open-coded blocks before, which is why one growing a call the other lacked was invisible. Behaviour deliberately preserved rather than harmonised: Telegram still records its partial verdict when the flow errors and healthcheck still does not. The rule is now stated in code, and pinned by a test with a populated result — the only way to pin it, since RunHealthcheckSetup returns a zero result on every error path today. Verified by mutation: 8 mutations, all caught. Two rounds were needed — the first M-b was caught by a build error rather than an assertion, and M-f survived because the test fed a zero-value result where the branch is indistinguishable. CLI stdout and every golden transcript are unchanged. --- cmd/proxsave/install_tui.go | 33 +-- cmd/proxsave/install_tui_setup_log.go | 92 +++++++ cmd/proxsave/install_tui_setup_log_test.go | 224 ++++++++++++++++++ cmd/proxsave/telegram_setup_cli.go | 11 +- .../orchestrator/telegram_setup_log_test.go | 68 ++++++ .../orchestrator/telegram_setup_sanitize.go | 23 ++ internal/ui/flows/install/telegram.go | 10 +- .../install/telegram_status_scrub_test.go | 81 +++++++ 8 files changed, 504 insertions(+), 38 deletions(-) create mode 100644 cmd/proxsave/install_tui_setup_log.go create mode 100644 cmd/proxsave/install_tui_setup_log_test.go create mode 100644 internal/orchestrator/telegram_setup_log_test.go create mode 100644 internal/ui/flows/install/telegram_status_scrub_test.go diff --git a/cmd/proxsave/install_tui.go b/cmd/proxsave/install_tui.go index d516c289..10cb810d 100644 --- a/cmd/proxsave/install_tui.go +++ b/cmd/proxsave/install_tui.go @@ -270,23 +270,7 @@ func runInstallTUI(ctx context.Context, configPath string, bootstrap *logging.Bo // returns Shown=false without any UI when Telegram is not centrally enabled. if !skipConfigWizard { telegramRes, telegramErr := flowinstall.RunTelegramSetup(ctx, session, baseDir, configPath, false) - if telegramErr != nil && bootstrap != nil { - bootstrap.Warning("Telegram setup failed (non-blocking): %v", telegramErr) - } - if bootstrap != nil && telegramErr == nil { - logTelegramSetupBootstrapOutcome(bootstrap, telegramRes.TelegramSetupBootstrap) - } - if bootstrap != nil && telegramRes.Shown { - if telegramRes.Verified { - bootstrap.Info("Telegram setup: verified (code=%d)", telegramRes.LastStatusCode) - } else if telegramRes.SkippedVerification { - bootstrap.Info("Telegram setup: verification skipped by user") - } else if telegramRes.CheckAttempts > 0 { - bootstrap.Info("Telegram setup: not verified (attempts=%d last=%d %s)", telegramRes.CheckAttempts, telegramRes.LastStatusCode, telegramRes.LastStatusMessage) - } else { - bootstrap.Info("Telegram setup: not verified (no check performed)") - } - } + logTelegramSetupOutcome(bootstrap, telegramRes, telegramErr) // Self-mode healthchecks: collect the ping URLs BEFORE the healthcheck // bootstrap re-reads the config (ordering invariant - eligibility keys off the @@ -303,20 +287,7 @@ func runInstallTUI(ctx context.Context, configPath string, bootstrap *logging.Bo // (self) verify the pasted alive URL is reachable. Eligibility is decided solely // by RunHealthcheckSetup (re-reads the written config); Shown=false with no UI otherwise. hcRes, hcErr := flowinstall.RunHealthcheckSetup(ctx, session, baseDir, configPath, false) - if hcErr != nil && bootstrap != nil { - bootstrap.Warning("Healthcheck setup failed (non-blocking): %v", hcErr) - } - if bootstrap != nil && hcErr == nil && hcRes.Shown { - if hcRes.Verified { - bootstrap.Info("Healthcheck setup: verified") - } else if hcRes.SkippedVerification { - bootstrap.Info("Healthcheck setup: check skipped by user") - } else if hcRes.CheckAttempts > 0 { - bootstrap.Info("Healthcheck setup: not verified (attempts=%d)", hcRes.CheckAttempts) - } else { - bootstrap.Info("Healthcheck setup: not verified (no check performed)") - } - } + logHealthcheckSetupOutcome(bootstrap, hcRes, hcErr) } // All interactive steps are done. Unlike the CLI, the TUI keeps the ALTSCREEN diff --git a/cmd/proxsave/install_tui_setup_log.go b/cmd/proxsave/install_tui_setup_log.go new file mode 100644 index 00000000..a63fe9c5 --- /dev/null +++ b/cmd/proxsave/install_tui_setup_log.go @@ -0,0 +1,92 @@ +package main + +import ( + "github.com/tis24dev/proxsave/internal/installer" + "github.com/tis24dev/proxsave/internal/logging" + "github.com/tis24dev/proxsave/internal/orchestrator" +) + +// The install log trail for the two optional setup steps, TUI side. +// +// The CLI emits these lines inline as it walks the step (runTelegramSetupCLI, +// runHealthcheckSetupCLI); the TUI cannot, because the step runs inside a flow that +// returns a result struct. Doing it from the driver is fine — what was NOT fine is +// that the driver open-coded the trail twice and got the two halves out of step: the +// Telegram half logged the bootstrap eligibility diagnosis, the healthcheck half +// silently dropped it. Keeping both here, in one shape, is what makes the omission +// visible the next time a step is added. + +// logTelegramSetupOutcome writes the install-log trail for one Telegram setup step: +// the non-blocking failure warning, the bootstrap eligibility diagnosis, and the +// verification verdict — the same three the CLI emits. +// +// The verdict lines are emitted even when err is non-nil: RunTelegramSetup returns +// what it collected before failing, and that partial verdict is worth recording. The +// eligibility diagnosis is not, because a run that failed may never have produced one. +func logTelegramSetupOutcome(bootstrap *logging.BootstrapLogger, res installer.TelegramSetupResult, err error) { + if bootstrap == nil { + return + } + if err != nil { + bootstrap.Warning("Telegram setup failed (non-blocking): %v", err) + } else { + logTelegramSetupBootstrapOutcome(bootstrap, res.TelegramSetupBootstrap) + } + if !res.Shown { + return + } + switch { + case res.Verified: + bootstrap.Info("Telegram setup: verified (code=%d)", res.LastStatusCode) + case res.SkippedVerification: + bootstrap.Info("Telegram setup: verification skipped by user") + case res.CheckAttempts > 0: + // The relay's own words, scrubbed and with the shared stand-in — the exact + // treatment runTelegramSetupCLI gives them. The field is already scrubbed at + // its write site; this call is what supplies the stand-in when the relay sent + // nothing, and it keeps the line identical to the CLI's if that ever changes. + bootstrap.Info("Telegram setup: not verified (attempts=%d last=%d %s)", + res.CheckAttempts, res.LastStatusCode, + orchestrator.TelegramSetupStatusMessageForLog(res.LastStatusMessage)) + default: + bootstrap.Info("Telegram setup: not verified (no check performed)") + } +} + +// logHealthcheckSetupOutcome is the healthcheck twin. It calls +// logHealthcheckSetupBootstrapOutcome, which the TUI install path did not: a skip the +// CLI explains in the log ("no alive URL configured yet", "unable to load config", the +// identity/secret verdict, "self mode") left no trace at all on a TUI install, so the +// step read as if it had never run and the reason it was skipped was unrecoverable +// from the log afterwards. +// +// Unlike the Telegram twin the verdict lines are suppressed on error. That is the rule +// the driver already applied here (`hcErr == nil &&` guarded the whole block), kept +// rather than harmonised because harmonising would be a behaviour change nobody asked +// for. It makes no observable difference today — RunHealthcheckSetup returns a ZERO +// result on every error path, and a zero result carries Shown=false and stops here +// anyway — so the rule is stated in code and pinned by a test with a populated result, +// which is the only way it can be pinned at all. +func logHealthcheckSetupOutcome(bootstrap *logging.BootstrapLogger, res installer.HealthcheckSetupResult, err error) { + if bootstrap == nil { + return + } + if err != nil { + bootstrap.Warning("Healthcheck setup failed (non-blocking): %v", err) + return + } + logHealthcheckSetupBootstrapOutcome(bootstrap, res.HealthcheckSetupBootstrap) + if !res.Shown { + return + } + switch { + case res.Verified: + bootstrap.Info("Healthcheck setup: verified") + case res.SkippedVerification: + bootstrap.Info("Healthcheck setup: check skipped by user") + case res.CheckAttempts > 0: + bootstrap.Info("Healthcheck setup: not verified (attempts=%d)", res.CheckAttempts) + default: + bootstrap.Info("Healthcheck setup: not verified (no check performed)") + } +} diff --git a/cmd/proxsave/install_tui_setup_log_test.go b/cmd/proxsave/install_tui_setup_log_test.go new file mode 100644 index 00000000..4273b16f --- /dev/null +++ b/cmd/proxsave/install_tui_setup_log_test.go @@ -0,0 +1,224 @@ +package main + +import ( + "bytes" + "errors" + "go/ast" + "go/parser" + "go/token" + "strings" + "testing" + + "github.com/tis24dev/proxsave/internal/installer" + "github.com/tis24dev/proxsave/internal/logging" + "github.com/tis24dev/proxsave/internal/orchestrator" + "github.com/tis24dev/proxsave/internal/types" +) + +// captureBootstrapLog returns a bootstrap logger whose lines are readable, mirroring +// the persisted install log: the mirror is exactly the sink startFlowSessionLog +// installs in production, so what lands in the buffer here is what lands on disk there. +func captureBootstrapLog(t *testing.T) (*logging.BootstrapLogger, *bytes.Buffer) { + t.Helper() + bootstrap := logging.NewBootstrapLogger() + buf := &bytes.Buffer{} + mirror := logging.New(types.LogLevelDebug, false) + mirror.SetOutput(buf) + bootstrap.SetMirrorLogger(mirror) + return bootstrap, buf +} + +// TestLogHealthcheckSetupOutcomeRecordsWhyTheStepWasSkipped is the regression proper. +// The TUI install driver never called logHealthcheckSetupBootstrapOutcome, so every +// reason the step can decline to run — no alive URL yet, unreadable config, no server +// identity, self mode — was absent from a TUI install log while the CLI recorded it. +// The step then looked like it had simply never happened, and the reason was gone for +// good, since nothing re-derives it after the install. +func TestLogHealthcheckSetupOutcomeRecordsWhyTheStepWasSkipped(t *testing.T) { + cases := []struct { + name string + eligibility orchestrator.HealthcheckSetupEligibility + want string + }{ + {"self mode without a URL", orchestrator.HealthcheckSetupSkipSelfMode, "no alive URL configured yet"}, + {"self mode", orchestrator.HealthcheckSetupEligibleSelf, "self mode"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + bootstrap, buf := captureBootstrapLog(t) + res := installer.HealthcheckSetupResult{ + HealthcheckSetupBootstrap: orchestrator.HealthcheckSetupBootstrap{Eligibility: tc.eligibility}, + } + logHealthcheckSetupOutcome(bootstrap, res, nil) + if !strings.Contains(buf.String(), tc.want) { + t.Fatalf("the install log must say why the healthcheck step did not run; want %q, got %q", tc.want, buf.String()) + } + }) + } + + // A config that cannot be read is the case where the log is the ONLY evidence: the + // step is skipped and the error is not surfaced anywhere else on the TUI path. + t.Run("unreadable config", func(t *testing.T) { + bootstrap, buf := captureBootstrapLog(t) + logHealthcheckSetupOutcome(bootstrap, installer.HealthcheckSetupResult{ + HealthcheckSetupBootstrap: orchestrator.HealthcheckSetupBootstrap{ + Eligibility: orchestrator.HealthcheckSetupSkipConfigError, + ConfigError: "permission denied reading backup.env", + }, + }, nil) + if !strings.Contains(buf.String(), "permission denied reading backup.env") { + t.Fatalf("the config error must reach the install log, got %q", buf.String()) + } + }) +} + +// TestLogHealthcheckSetupOutcomeStaysSilentOnBootstrapFailure pins the asymmetry with +// the Telegram twin, which the driver carried before this refactor: on error the +// healthcheck step gets a warning and NOTHING else, while Telegram still records its +// partial verdict. +// +// The result passed here is deliberately NOT the zero value. RunHealthcheckSetup +// happens to return a zero result on every error path today, which makes the choice +// invisible in production and unpinnable with a realistic fixture — a zero result +// carries Shown=false and exits on its own. Feeding a populated result is what makes +// the rule itself testable, so a later change that starts returning partial results on +// error cannot silently flip this behaviour. +func TestLogHealthcheckSetupOutcomeStaysSilentOnBootstrapFailure(t *testing.T) { + bootstrap, buf := captureBootstrapLog(t) + logHealthcheckSetupOutcome(bootstrap, installer.HealthcheckSetupResult{ + HealthcheckSetupBootstrap: orchestrator.HealthcheckSetupBootstrap{ + Eligibility: orchestrator.HealthcheckSetupSkipSelfMode, + }, + Shown: true, CheckAttempts: 2, + }, errors.New("boom")) + + out := buf.String() + if !strings.Contains(out, "Healthcheck setup failed (non-blocking): boom") { + t.Fatalf("the failure must be recorded, got %q", out) + } + if strings.Contains(out, "not verified") { + t.Fatalf("a failed step must not get a verdict line, got %q", out) + } + if strings.Contains(out, "no alive URL configured yet") { + t.Fatalf("a failed step must not get an eligibility diagnosis either, got %q", out) + } +} + +// TestLogTelegramSetupOutcomeScrubsTheRelayMessage: the relay's own words go into the +// persisted install log, and that log is read back with cat/less later. Terminal +// escapes surviving into it let a hostile relay response drive the reader's terminal. +// The CLI has scrubbed here from the start; the TUI logged the raw bytes. +func TestLogTelegramSetupOutcomeScrubsTheRelayMessage(t *testing.T) { + bootstrap, buf := captureBootstrapLog(t) + logTelegramSetupOutcome(bootstrap, installer.TelegramSetupResult{ + Shown: true, + CheckAttempts: 3, + LastStatusCode: 409, + LastStatusMessage: "\x1b[2Jnot linked\x07", + }, nil) + + out := buf.String() + if strings.ContainsAny(out, "\x1b\x07") { + t.Fatalf("the install log must not carry the relay's control bytes: %q", out) + } + if !strings.Contains(out, "Telegram setup: not verified (attempts=3 last=409 not linked)") { + t.Fatalf("the scrubbed relay message must still reach the log: %q", out) + } +} + +// TestLogTelegramSetupOutcomeStandsInForASilentRelay: without the stand-in the line +// ends on a bare status code, which reads as a truncated log rather than a relay that +// said nothing. Same stand-in the CLI writes, so the two install logs match. +func TestLogTelegramSetupOutcomeStandsInForASilentRelay(t *testing.T) { + bootstrap, buf := captureBootstrapLog(t) + logTelegramSetupOutcome(bootstrap, installer.TelegramSetupResult{ + Shown: true, CheckAttempts: 1, LastStatusCode: 500, LastStatusMessage: "", + }, nil) + want := "Telegram setup: not verified (attempts=1 last=500 " + orchestrator.TelegramSetupStatusUnknownMessage + ")" + if !strings.Contains(buf.String(), want) { + t.Fatalf("want %q in the log, got %q", want, buf.String()) + } +} + +// TestLogTelegramSetupOutcomeKeepsThePartialVerdictOnFailure pins the asymmetry with +// the healthcheck twin: RunTelegramSetup returns what it collected before failing, so +// the verdict is real and worth recording — only the eligibility diagnosis, which a +// failed run may never have produced, is dropped. +func TestLogTelegramSetupOutcomeKeepsThePartialVerdictOnFailure(t *testing.T) { + bootstrap, buf := captureBootstrapLog(t) + logTelegramSetupOutcome(bootstrap, installer.TelegramSetupResult{ + TelegramSetupBootstrap: orchestrator.TelegramSetupBootstrap{ + Eligibility: orchestrator.TelegramSetupSkipPersonalMode, + }, + Shown: true, SkippedVerification: true, + }, errors.New("session died")) + + out := buf.String() + if !strings.Contains(out, "Telegram setup failed (non-blocking): session died") { + t.Fatalf("the failure must be recorded, got %q", out) + } + if !strings.Contains(out, "Telegram setup: verification skipped by user") { + t.Fatalf("the partial verdict must survive the failure, got %q", out) + } + if strings.Contains(out, "personal mode selected") { + t.Fatalf("the eligibility diagnosis must be dropped on failure, got %q", out) + } +} + +// TestSetupOutcomeLoggersTolerateANilBootstrap: the driver passes the bootstrap logger +// straight through and it is nil on paths that keep no install log. Both used to be +// guarded by an `if bootstrap != nil` at every call site; the guard now lives inside. +func TestSetupOutcomeLoggersTolerateANilBootstrap(t *testing.T) { + logTelegramSetupOutcome(nil, installer.TelegramSetupResult{Shown: true}, errors.New("boom")) + logHealthcheckSetupOutcome(nil, installer.HealthcheckSetupResult{Shown: true}, errors.New("boom")) +} + +// TestInstallTUIDriverLogsBothSetupOutcomes pins the invariant that was actually +// broken, which none of the tests above can reach: the driver called the Telegram +// logger and simply did not call the healthcheck one. Both were open-coded blocks +// then, so the omission was invisible; they are one call each now, but a call is +// still something a future edit can drop, and dropping it is silent — the install +// succeeds, only the log is poorer, and no run fails. +// +// This is a STRUCTURAL test and it knows it: it asserts the driver contains the two +// calls, not what they emit (that is the rest of this file). Driving runInstallTUI +// itself would mean a session, a written config and a relay stub for one bit of +// information. If the driver is ever restructured so these calls move or are made +// through a seam, replace this test rather than deleting it. +func TestInstallTUIDriverLogsBothSetupOutcomes(t *testing.T) { + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, "install_tui.go", nil, 0) + if err != nil { + t.Fatalf("parsing the install driver: %v", err) + } + + var driver *ast.FuncDecl + ast.Inspect(file, func(n ast.Node) bool { + if fn, ok := n.(*ast.FuncDecl); ok && fn.Name.Name == "runInstallTUI" { + driver = fn + return false + } + return true + }) + if driver == nil { + t.Fatal("runInstallTUI not found in install_tui.go") + } + + called := map[string]bool{} + ast.Inspect(driver, func(n ast.Node) bool { + call, ok := n.(*ast.CallExpr) + if !ok { + return true + } + if ident, ok := call.Fun.(*ast.Ident); ok { + called[ident.Name] = true + } + return true + }) + + for _, name := range []string{"logTelegramSetupOutcome", "logHealthcheckSetupOutcome"} { + if !called[name] { + t.Errorf("runInstallTUI does not call %s, so that step's diagnosis never reaches the install log", name) + } + } +} diff --git a/cmd/proxsave/telegram_setup_cli.go b/cmd/proxsave/telegram_setup_cli.go index b359f6e9..59f033db 100644 --- a/cmd/proxsave/telegram_setup_cli.go +++ b/cmd/proxsave/telegram_setup_cli.go @@ -99,12 +99,11 @@ func runTelegramSetupCLI(ctx context.Context, reader *bufio.Reader, baseDir, con return nil } - // rawMsg drives ONLY the byte-identical failure log line (sanitized RAW - // server message). The user-facing line is always the classifier message. - rawMsg := sanitizeTelegramSetupStatusMessage(res.Status.Message) - if rawMsg == "" { - rawMsg = "Registration not active yet" - } + // rawMsg drives ONLY the failure log line (the scrubbed RAW server message, + // with the shared stand-in when scrubbing leaves nothing). The user-facing + // line is always the classifier message. Routed through the shared helper so + // the TUI install log, which now calls the same one, cannot drift from this. + rawMsg := orchestrator.TelegramSetupStatusMessageForLog(res.Status.Message) fmt.Printf("Telegram: %s\n", sanitizeTelegramSetupStatusMessage(st.Message)) if st.Fatal { // 422 / 426: re-checking cannot help, do NOT offer "Check again?" diff --git a/internal/orchestrator/telegram_setup_log_test.go b/internal/orchestrator/telegram_setup_log_test.go new file mode 100644 index 00000000..f297abe1 --- /dev/null +++ b/internal/orchestrator/telegram_setup_log_test.go @@ -0,0 +1,68 @@ +package orchestrator + +import ( + "strings" + "testing" +) + +// TestTelegramSetupStatusMessageForLog pins the three things the persisted install log +// needs from an untrusted relay message, and which every front-end must apply the same +// way: control sequences stripped, length capped, and a stand-in when nothing is left. +// Only the first is about safety; the other two are why a shared helper exists at all, +// since a front-end that remembered to scrub but not to substitute would write a line +// ending on a bare status code. +func TestTelegramSetupStatusMessageForLog(t *testing.T) { + t.Run("strips terminal escapes", func(t *testing.T) { + got := TelegramSetupStatusMessageForLog(" \x1b[31mnot\tlinked\r\nyet\x1b[0m\x07 ") + if got != "not linked yet" { + t.Fatalf("got %q, want %q", got, "not linked yet") + } + }) + + // The stand-in is for a relay that said NOTHING. A relay that sent only control + // bytes said something, and the sanitizer already answers that with an + // ASCII-quoted rendering — which is strictly more useful in a log than the + // stand-in, because it preserves what arrived while making it safe to print. + // Substituting there would destroy the only record of a malformed response. + t.Run("stands in only for a silent relay", func(t *testing.T) { + for _, raw := range []string{"", " ", "\t\n"} { + if got := TelegramSetupStatusMessageForLog(raw); got != TelegramSetupStatusUnknownMessage { + t.Fatalf("TelegramSetupStatusMessageForLog(%q) = %q, want the stand-in %q", + raw, got, TelegramSetupStatusUnknownMessage) + } + } + for _, raw := range []string{"\x1b[2J\x1b[H", "\x00\x01\x02"} { + got := TelegramSetupStatusMessageForLog(raw) + if got == TelegramSetupStatusUnknownMessage { + t.Fatalf("a control-only response must be quoted, not replaced by the stand-in: %q", raw) + } + if strings.ContainsAny(got, "\x1b\x00\x01\x02") { + t.Fatalf("the quoted rendering must still be printable: %q", got) + } + } + }) + + t.Run("caps the length", func(t *testing.T) { + got := TelegramSetupStatusMessageForLog(strings.Repeat("é", TelegramSetupStatusMessageMaxRunes*3)) + if n := len([]rune(got)); n > TelegramSetupStatusMessageMaxRunes { + t.Fatalf("got %d runes, want at most %d", n, TelegramSetupStatusMessageMaxRunes) + } + }) + + // Idempotence is load-bearing, not incidental: the TUI scrubs at the write site and + // then routes the field through here for the stand-in, so a second pass must not + // truncate an already-truncated message again or re-escape it. + t.Run("is idempotent", func(t *testing.T) { + for _, raw := range []string{ + "not linked yet", + " \x1b[31mnot linked\x1b[0m ", + strings.Repeat("x", TelegramSetupStatusMessageMaxRunes*2), + "", + } { + once := TelegramSetupStatusMessageForLog(raw) + if twice := TelegramSetupStatusMessageForLog(once); twice != once { + t.Fatalf("second pass over %q changed %q into %q", raw, once, twice) + } + } + }) +} diff --git a/internal/orchestrator/telegram_setup_sanitize.go b/internal/orchestrator/telegram_setup_sanitize.go index 4d6e8b69..b58b1f64 100644 --- a/internal/orchestrator/telegram_setup_sanitize.go +++ b/internal/orchestrator/telegram_setup_sanitize.go @@ -32,6 +32,29 @@ func SanitizeTelegramSetupStatusMessage(raw string) string { return TruncateTelegramSetupStatusMessage(quoted) } +// TelegramSetupStatusUnknownMessage stands in for the relay's status message in the +// "not verified" log line when the server said nothing usable — either it sent no +// message at all, or everything it sent was stripped as a control sequence. Without +// it the line ends on a bare status code with nothing after it, which reads like a +// log that was cut short rather than a server that stayed silent. +const TelegramSetupStatusUnknownMessage = "Registration not active yet" + +// TelegramSetupStatusMessageForLog is the untrusted relay status message as EVERY +// front-end must write it into the persisted install log: scrubbed, truncated, and +// replaced by the stand-in when nothing survives. That log is written to disk and +// read back later, so an escape sequence surviving into it would move the cursor of +// whoever cats the file — the CLI scrubbed here from the start, the TUI did not. +// +// Sanitizing is idempotent (a scrubbed string has no control bytes left to strip and +// is already within the cap), so a caller that already scrubbed at the write site can +// still route through here to pick up the stand-in. +func TelegramSetupStatusMessageForLog(raw string) string { + if msg := SanitizeTelegramSetupStatusMessage(raw); msg != "" { + return msg + } + return TelegramSetupStatusUnknownMessage +} + func stripTelegramTerminalSequences(msg string) string { var b strings.Builder b.Grow(len(msg)) diff --git a/internal/ui/flows/install/telegram.go b/internal/ui/flows/install/telegram.go index a79e967e..6ac8ba97 100644 --- a/internal/ui/flows/install/telegram.go +++ b/internal/ui/flows/install/telegram.go @@ -89,7 +89,15 @@ func RunTelegramSetup(ctx context.Context, session *shell.Session, baseDir, conf if !cancelled { result.CheckAttempts++ result.LastStatusCode = res.Status.Code - result.LastStatusMessage = res.Status.Message // RAW preserved (parity with tview) + // Scrubbed at the WRITE site. This field exists to be logged — + // the install driver puts it in the persisted bootstrap log — and + // it carries whatever the relay chose to send, so scrubbing here + // is what keeps terminal escapes out of every consumer instead of + // trusting each one to remember. It previously held the raw bytes + // ("parity with tview") and the install log inherited them. The + // status line the user reads is st.Message below, which the + // classifier scrubs separately. + result.LastStatusMessage = orchestrator.SanitizeTelegramSetupStatusMessage(res.Status.Message) if res.Status.Error != nil { result.LastStatusError = res.Status.Error.Error() } else { diff --git a/internal/ui/flows/install/telegram_status_scrub_test.go b/internal/ui/flows/install/telegram_status_scrub_test.go new file mode 100644 index 00000000..a32fabf7 --- /dev/null +++ b/internal/ui/flows/install/telegram_status_scrub_test.go @@ -0,0 +1,81 @@ +package install + +import ( + "context" + "strings" + "testing" + + "github.com/tis24dev/proxsave/internal/installer" + "github.com/tis24dev/proxsave/internal/logging" + "github.com/tis24dev/proxsave/internal/notify" + "github.com/tis24dev/proxsave/internal/orchestrator" +) + +// TestRunTelegramSetupScrubsTheRelayStatusMessage: LastStatusMessage exists to be +// written into the persisted install log, and it carries a string the relay chose. It +// used to be stored raw ("parity with tview"), so a relay that answered with terminal +// escapes had them land in a file that is later cat'd by whoever debugs the install. +// Scrubbing belongs at THIS write site rather than at the log site, because the field +// is what future consumers will reach for and none of them can be relied on to +// remember. Pinned here so a reader who sees a scrub at the log site cannot conclude +// the field itself may go back to raw. +func TestRunTelegramSetupScrubsTheRelayStatusMessage(t *testing.T) { + d := newDriver(t) + + origBootstrap := telegramBuildBootstrap + origCheck := telegramCheckRegistration + t.Cleanup(func() { + telegramBuildBootstrap = origBootstrap + telegramCheckRegistration = origCheck + }) + + telegramBuildBootstrap = func(configPath, baseDir string) (orchestrator.TelegramSetupBootstrap, error) { + return orchestrator.TelegramSetupBootstrap{ + Eligibility: orchestrator.TelegramSetupEligibleCentralized, + ServerID: "12345678", + }, nil + } + // A hostile-but-plausible 409: a clear-screen sequence, a cursor jump, a bell and a + // carriage return wrapped around otherwise ordinary copy. + const hostile = "\x1b[2J\x1b[1;1Hnot\tlinked\r\nyet\x07" + telegramCheckRegistration = func(ctx context.Context, host, serverID, baseDir string, logger *logging.Logger) notify.TelegramRegistrationResult { + res := notify.TelegramRegistrationResult{} + res.Status.Code = 409 + res.Status.Message = hostile + return res + } + + type outcome struct { + res installer.TelegramSetupResult + err error + } + resCh := make(chan outcome, 1) + go func() { + res, err := RunTelegramSetup(context.Background(), d.session, t.TempDir(), "/tmp/backup.env", false) + resCh <- outcome{res, err} + }() + + d.waitScreen("Telegram setup") + d.keys("enter") // Check + d.waitScreen("Telegram setup") + d.keys("down enter") // Skip (409 is not verified, so the leave action is Skip) + got := <-resCh + if got.err != nil { + t.Fatalf("RunTelegramSetup: %v", got.err) + } + if got.res.CheckAttempts != 1 || got.res.LastStatusCode != 409 { + t.Fatalf("test setup: the check did not run as expected: %+v", got.res) + } + + msg := got.res.LastStatusMessage + if strings.ContainsAny(msg, "\x1b\x07\r\n\t") { + t.Fatalf("LastStatusMessage still carries control bytes and would reach the install log: %q", msg) + } + if want := orchestrator.SanitizeTelegramSetupStatusMessage(hostile); msg != want { + t.Fatalf("LastStatusMessage = %q, want the shared sanitizer's output %q", msg, want) + } + // The words survive: scrubbing must not cost the reader the reason. + if !strings.Contains(msg, "not linked yet") { + t.Fatalf("LastStatusMessage lost the relay's actual message: %q", msg) + } +} From 6ead3f0402fa6283ceb7ba7c069e35c298bed401 Mon Sep 17 00:00:00 2001 From: tis24dev Date: Mon, 3 Aug 2026 04:03:40 +0200 Subject: [PATCH 11/50] fix(guards): --cleanup-guards no longer exits 0 while the storage is still locked MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The CLI took orchestrator.CleanupMountGuards, the error-only wrapper, so anything short of an engine failure exited 0 — including a run that removed nothing because the guards sit under a live mount. A script gating on the exit code was told the storage was unlocked while it was still held. The dashboard, driving the same engine through CleanupMountGuardsReport, classified that same report as PENDING and coloured it yellow. The CLI now takes the report too, through the seam the dashboard already uses. New exit code ExitGuardsPending (17), deliberately distinct from ExitGenericError. Like ExitBackupSkipped it marks a STATE, not a failure: nothing went wrong, the work cannot finish until the datastore is offline. A script needs the two apart, because "report a bug" and "unmount and retry" are different remedies. GuardsRemaining == -1 — the unknown sentinel — counts as pending, the same fail-closed rule guardApplyClean already applied on the graphical side. --cleanup-guards --dry-run now prints the CLEAN/FOUND verdict and reflects it in the exit code, making it usable as a read-only probe. It already performed exactly that read and threw the report away. The classification and the FACTS are now shared by both front-ends; the call to action is not, and must not be: the dashboard names a button ("Apply"), the CLI names a flag. A test pins that the shared facts stay free of either. Verified by mutation: 8 mutations, all caught. Two needed a second round — M-d was anchored on a string that appears twice in main_modes.go, and M-e survived because the test it was paired with called the verdict function directly and so could not notice the mode dropping the call, which is what the mode used to do. It now captures real stdout. --- cmd/proxsave/cleanup_guards_verdict.go | 100 ++++++++ cmd/proxsave/cleanup_guards_verdict_test.go | 270 ++++++++++++++++++++ cmd/proxsave/dashboard_cleanup_guards.go | 38 +-- cmd/proxsave/main_modes.go | 13 +- internal/types/exit_codes.go | 12 + 5 files changed, 401 insertions(+), 32 deletions(-) create mode 100644 cmd/proxsave/cleanup_guards_verdict.go create mode 100644 cmd/proxsave/cleanup_guards_verdict_test.go diff --git a/cmd/proxsave/cleanup_guards_verdict.go b/cmd/proxsave/cleanup_guards_verdict.go new file mode 100644 index 00000000..2209b13c --- /dev/null +++ b/cmd/proxsave/cleanup_guards_verdict.go @@ -0,0 +1,100 @@ +package main + +import ( + "fmt" + "strings" + + "github.com/tis24dev/proxsave/internal/logging" + "github.com/tis24dev/proxsave/internal/orchestrator" + "github.com/tis24dev/proxsave/internal/types" +) + +// The guard-cleanup verdict, single-sourced for BOTH front-ends. +// +// Both drive orchestrator.CleanupMountGuardsReport, but they used to read it +// differently: the dashboard classified the report (CLEAN/FOUND, DONE/PENDING) while +// --cleanup-guards took the error-only CleanupMountGuards wrapper and exited 0 for +// anything short of an engine failure. A script gating on the exit code was told the +// storage was unlocked while guards were still holding it. +// +// What is shared here is the CLASSIFICATION and the FACTS. The call to action is not: +// the dashboard names a button ("Apply"), the CLI names a flag, and neither wording +// makes sense on the other front-end. + +// guardApplyClean reports whether a real run left nothing behind. GuardsRemaining == -1 +// is the fail-closed "unknown" sentinel, which counts as NOT clean: a cleanup that +// cannot confirm what remains must not be reported as having unlocked the storage. +func guardApplyClean(r orchestrator.GuardCleanupReport) bool { + return r.GuardsRemaining == 0 && r.ImmutablePending == 0 +} + +// guardCleanupExitCode maps a report to the process exit code. dryRun selects the +// read-only CHECK rule (anything found is reported as pending, since nothing was +// removed) over the real-run rule (only what is LEFT counts). +// +// Both non-clean outcomes exit ExitGuardsPending rather than ExitGenericError: the +// cleanup did not fail, the storage is simply still locked. An engine error is the +// caller's to report, and keeps ExitGenericError. +func guardCleanupExitCode(r orchestrator.GuardCleanupReport, dryRun bool) types.ExitCode { + if dryRun { + if r.HasGuards() { + return types.ExitGuardsPending + } + return types.ExitSuccess + } + if guardApplyClean(r) { + return types.ExitSuccess + } + return types.ExitGuardsPending +} + +// guardCheckFacts states what the read-only check found, and nothing about what to do +// next — see the note at the top of this file. +func guardCheckFacts(r orchestrator.GuardCleanupReport) string { + if !r.HasGuards() { + return "No restore mount guards are present. Nothing to unlock." + } + var parts []string + if r.BindGuards > 0 { + parts = append(parts, countLabel(r.BindGuards, "bind mount guard")) + } + if r.ImmutableGuards > 0 { + parts = append(parts, countLabel(r.ImmutableGuards, "immutable flag")) + } + return fmt.Sprintf("Found %s locking the storage.", strings.Join(parts, " and ")) +} + +// guardApplyFacts states the outcome of a real run. The not-clean wording deliberately +// names the usual cause — a guard hidden under a live mount, which the engine refuses +// to unmount — because that is what tells the operator the retry needs the datastore +// offline rather than more privileges. +func guardApplyFacts(r orchestrator.GuardCleanupReport) string { + if guardApplyClean(r) { + return "Removed the restore mount guards. The storage is unlocked." + } + return "Some guards are still in place (hidden under a live mount)." +} + +// logCLIGuardVerdict states the verdict in the CLI's voice: the shared facts plus a +// call to action naming the flag. Warning level for anything left behind, so it stands +// out in a cron log — the place this mode usually runs, and where nobody is watching. +func logCLIGuardVerdict(logger *logging.Logger, r orchestrator.GuardCleanupReport, dryRun bool) { + switch { + case dryRun && r.HasGuards(): + logger.Warning("%s Run without --dry-run to remove them.", guardCheckFacts(r)) + case dryRun: + logger.Info("%s", guardCheckFacts(r)) + case guardApplyClean(r): + logger.Info("%s", guardApplyFacts(r)) + default: + logger.Warning("%s Unmount the datastore and run --cleanup-guards again once it is offline.", guardApplyFacts(r)) + } +} + +// countLabel pluralizes "N thing" / "N things". +func countLabel(n int, singular string) string { + if n == 1 { + return fmt.Sprintf("%d %s", n, singular) + } + return fmt.Sprintf("%d %ss", n, singular) +} diff --git a/cmd/proxsave/cleanup_guards_verdict_test.go b/cmd/proxsave/cleanup_guards_verdict_test.go new file mode 100644 index 00000000..083ab6b7 --- /dev/null +++ b/cmd/proxsave/cleanup_guards_verdict_test.go @@ -0,0 +1,270 @@ +package main + +import ( + "bytes" + "context" + "errors" + "strings" + "testing" + + "github.com/tis24dev/proxsave/internal/cli" + "github.com/tis24dev/proxsave/internal/logging" + "github.com/tis24dev/proxsave/internal/orchestrator" + "github.com/tis24dev/proxsave/internal/types" +) + +// TestCleanupGuardsExitCodeReportsPendingGuards is the regression proper: the CLI took +// the error-only wrapper and exited 0 for anything short of an engine failure, so a +// script gating on `proxsave --cleanup-guards` was told the storage was unlocked while +// guards were still holding it. Every non-clean outcome must now exit ExitGuardsPending +// — and it must stay distinct from ExitGenericError, because "the cleanup failed" and +// "the cleanup ran but the datastore is still mounted" need different remedies. +func TestCleanupGuardsExitCodeReportsPendingGuards(t *testing.T) { + cases := []struct { + name string + report orchestrator.GuardCleanupReport + dryRun bool + want types.ExitCode + }{ + { + name: "check finds guards", + report: orchestrator.GuardCleanupReport{BindGuards: 2}, + dryRun: true, + want: types.ExitGuardsPending, + }, + { + name: "check finds nothing", + report: orchestrator.GuardCleanupReport{}, + dryRun: true, + want: types.ExitSuccess, + }, + { + name: "run removes everything", + report: orchestrator.GuardCleanupReport{BindGuards: 2, Unmounted: 2, GuardsRemaining: 0}, + want: types.ExitSuccess, + }, + { + name: "run leaves a guard behind", + report: orchestrator.GuardCleanupReport{BindGuards: 2, Unmounted: 1, GuardsRemaining: 1}, + want: types.ExitGuardsPending, + }, + { + name: "run leaves an immutable flag pending", + report: orchestrator.GuardCleanupReport{ImmutableGuards: 1, ImmutablePending: 1}, + want: types.ExitGuardsPending, + }, + { + // -1 is the fail-closed unknown sentinel. A cleanup that cannot confirm + // what remains must not be reported as having unlocked the storage — the + // dashboard already treats it this way via guardApplyClean. + name: "run cannot confirm what remains", + report: orchestrator.GuardCleanupReport{BindGuards: 1, GuardsRemaining: -1}, + want: types.ExitGuardsPending, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := guardCleanupExitCode(tc.report, tc.dryRun); got != tc.want { + t.Fatalf("guardCleanupExitCode = %d (%s), want %d (%s)", got, got, tc.want, tc.want) + } + }) + } +} + +// TestRunCleanupGuardsModeReflectsTheVerdictInTheExitCode drives the mode end to end +// through the same seam the dashboard flow uses, so the exit code a caller actually +// observes is pinned, not just the classifier. +func TestRunCleanupGuardsModeReflectsTheVerdictInTheExitCode(t *testing.T) { + cases := []struct { + name string + report orchestrator.GuardCleanupReport + dryRun bool + want int + }{ + {"dry run, guards found", orchestrator.GuardCleanupReport{BindGuards: 1}, true, types.ExitGuardsPending.Int()}, + {"dry run, clean", orchestrator.GuardCleanupReport{}, true, types.ExitSuccess.Int()}, + {"real run, unlocked", orchestrator.GuardCleanupReport{BindGuards: 1, Unmounted: 1}, false, types.ExitSuccess.Int()}, + {"real run, still locked", orchestrator.GuardCleanupReport{BindGuards: 1, GuardsRemaining: 1}, false, types.ExitGuardsPending.Int()}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + stubGuardReport(t, tc.report, tc.report) + code, handled := runCleanupGuardsMode(context.Background(), + &cli.Args{CleanupGuards: true, DryRun: tc.dryRun}, logging.NewBootstrapLogger()) + if !handled { + t.Fatal("--cleanup-guards must be handled by this mode") + } + if code != tc.want { + t.Fatalf("exit code = %d, want %d", code, tc.want) + } + }) + } +} + +// TestRunCleanupGuardsModePrintsTheVerdict: the exit code is the machine-readable half +// of this fix and the verdict is the human half — an operator running the mode by hand +// needs to be told WHAT is still locking the storage, which the CLI never said. The +// mode builds its own logger, so this captures real stdout rather than calling +// logCLIGuardVerdict directly; a test that called it directly could not notice the mode +// dropping the call entirely, which is exactly what it used to do. +func TestRunCleanupGuardsModePrintsTheVerdict(t *testing.T) { + stubGuardReport(t, + orchestrator.GuardCleanupReport{BindGuards: 2}, + orchestrator.GuardCleanupReport{BindGuards: 2, GuardsRemaining: 2}) + + out := captureNewKeyStdout(t, func() { + runCleanupGuardsMode(context.Background(), //nolint:errcheck + &cli.Args{CleanupGuards: true, DryRun: true}, logging.NewBootstrapLogger()) + }) + for _, want := range []string{"2 bind mount guards", "locking the storage"} { + if !strings.Contains(out, want) { + t.Fatalf("the mode must print the verdict; want %q in %q", want, out) + } + } +} + +// TestRunCleanupGuardsModeKeepsEngineFailuresDistinct: an engine error is a failure to +// report, not a locked datastore to retry. Collapsing the two would defeat the reason +// ExitGuardsPending exists at all. +func TestRunCleanupGuardsModeKeepsEngineFailuresDistinct(t *testing.T) { + orig := cleanupGuardsReport + t.Cleanup(func() { cleanupGuardsReport = orig }) + cleanupGuardsReport = func(_ context.Context, _ *logging.Logger, _ bool) (orchestrator.GuardCleanupReport, error) { + return orchestrator.GuardCleanupReport{}, errors.New("mountinfo unreadable") + } + + code, handled := runCleanupGuardsMode(context.Background(), + &cli.Args{CleanupGuards: true}, logging.NewBootstrapLogger()) + if !handled { + t.Fatal("--cleanup-guards must be handled by this mode") + } + if code != types.ExitGenericError.Int() { + t.Fatalf("an engine failure must exit %d, got %d", types.ExitGenericError.Int(), code) + } + if code == types.ExitGuardsPending.Int() { + t.Fatal("an engine failure must not be reported as pending guards") + } +} + +// TestRunCleanupGuardsModeIsInertWithoutTheFlag: the mode dispatcher calls every +// runXxxMode in turn, so this one must decline cleanly — and must not touch the engine +// when it does. +func TestRunCleanupGuardsModeIsInertWithoutTheFlag(t *testing.T) { + orig := cleanupGuardsReport + t.Cleanup(func() { cleanupGuardsReport = orig }) + cleanupGuardsReport = func(_ context.Context, _ *logging.Logger, _ bool) (orchestrator.GuardCleanupReport, error) { + t.Fatal("the engine must not run without --cleanup-guards") + return orchestrator.GuardCleanupReport{}, nil + } + + code, handled := runCleanupGuardsMode(context.Background(), &cli.Args{}, logging.NewBootstrapLogger()) + if handled || code != types.ExitSuccess.Int() { + t.Fatalf("without the flag the mode must decline, got code=%d handled=%v", code, handled) + } +} + +// TestCLIGuardVerdictSaysWhatWasFoundAndWhatToDo: the exit code alone tells a script +// what happened but tells a human nothing. The CLI had no counterpart to the +// dashboard's CLEAN/FOUND verdict at all — the read was performed and the report +// discarded. +func TestCLIGuardVerdictSaysWhatWasFoundAndWhatToDo(t *testing.T) { + cases := []struct { + name string + report orchestrator.GuardCleanupReport + dryRun bool + want []string + notWant []string + }{ + { + name: "check with two kinds of guard", + report: orchestrator.GuardCleanupReport{BindGuards: 2, ImmutableGuards: 1}, + dryRun: true, + want: []string{"2 bind mount guards", "1 immutable flag", "locking the storage", "--dry-run"}, + }, + { + name: "check with nothing to unlock", + report: orchestrator.GuardCleanupReport{}, + dryRun: true, + want: []string{"Nothing to unlock"}, + notWant: []string{"--dry-run"}, // no action to suggest + }, + { + name: "real run leaves guards behind", + report: orchestrator.GuardCleanupReport{BindGuards: 1, GuardsRemaining: 1}, + want: []string{"still in place", "hidden under a live mount", "Unmount the datastore", "--cleanup-guards"}, + }, + { + name: "real run unlocks the storage", + report: orchestrator.GuardCleanupReport{BindGuards: 1, Unmounted: 1}, + want: []string{"storage is unlocked"}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + buf := &bytes.Buffer{} + logger := logging.New(types.LogLevelInfo, false) + logger.SetOutput(buf) + logCLIGuardVerdict(logger, tc.report, tc.dryRun) + + out := buf.String() + for _, want := range tc.want { + if !strings.Contains(out, want) { + t.Errorf("verdict must mention %q, got %q", want, out) + } + } + for _, notWant := range tc.notWant { + if strings.Contains(out, notWant) { + t.Errorf("verdict must not mention %q, got %q", notWant, out) + } + } + }) + } +} + +// TestCLIGuardVerdictWarnsWhenTheStorageStaysLocked: this mode usually runs from cron, +// where nobody is watching. A locked storage reported at INFO alongside every other +// line is a line nobody reads; the dashboard already colours it yellow. +func TestCLIGuardVerdictWarnsWhenTheStorageStaysLocked(t *testing.T) { + for _, tc := range []struct { + name string + report orchestrator.GuardCleanupReport + dryRun bool + }{ + {"guards found by the check", orchestrator.GuardCleanupReport{BindGuards: 1}, true}, + {"guards left by a real run", orchestrator.GuardCleanupReport{BindGuards: 1, GuardsRemaining: 1}, false}, + } { + t.Run(tc.name, func(t *testing.T) { + buf := &bytes.Buffer{} + logger := logging.New(types.LogLevelInfo, false) + logger.SetOutput(buf) + logCLIGuardVerdict(logger, tc.report, tc.dryRun) + if !strings.Contains(buf.String(), "WARNING") { + t.Fatalf("a storage that stays locked must be logged as a warning, got %q", buf.String()) + } + }) + } +} + +// TestGuardFactsCarryNoFrontEndCallToAction pins the split the shared helpers exist +// for: the dashboard names a button, the CLI names a flag, and neither wording may +// leak into the facts both of them render. +func TestGuardFactsCarryNoFrontEndCallToAction(t *testing.T) { + reports := []orchestrator.GuardCleanupReport{ + {}, + {BindGuards: 3}, + {ImmutableGuards: 2, ImmutablePending: 1}, + {BindGuards: 1, GuardsRemaining: -1}, + } + for _, r := range reports { + for _, facts := range []string{guardCheckFacts(r), guardApplyFacts(r)} { + for _, leak := range []string{"Apply", "--dry-run", "--cleanup-guards", "Cleanup guards"} { + if strings.Contains(facts, leak) { + t.Errorf("shared facts %q must not name a front-end action (%q)", facts, leak) + } + } + } + } +} diff --git a/cmd/proxsave/dashboard_cleanup_guards.go b/cmd/proxsave/dashboard_cleanup_guards.go index a035abf5..cd8a27cd 100644 --- a/cmd/proxsave/dashboard_cleanup_guards.go +++ b/cmd/proxsave/dashboard_cleanup_guards.go @@ -2,9 +2,7 @@ package main import ( "context" - "fmt" "io" - "strings" "github.com/tis24dev/proxsave/internal/logging" "github.com/tis24dev/proxsave/internal/orchestrator" @@ -60,40 +58,20 @@ func classifyGuardApply(r orchestrator.GuardCleanupReport) (orchestrator.Healthc return orchestrator.HealthcheckSetupLevelWarn, "PENDING" } -// guardApplyClean reports whether a real run left nothing behind. GuardsRemaining == -1 -// is the fail-closed "unknown" sentinel, which counts as not-clean. -func guardApplyClean(r orchestrator.GuardCleanupReport) bool { - return r.GuardsRemaining == 0 && r.ImmutablePending == 0 -} - -// describeGuardCheck renders the CHECK explanation (no "dry run" wording): either that -// there is nothing to unlock, or what was found locking the storage. +// describeGuardCheck renders the CHECK explanation (no "dry run" wording): the shared +// facts, plus this front-end's call to action, which names the on-screen button. func describeGuardCheck(r orchestrator.GuardCleanupReport) string { if !r.HasGuards() { - return "No restore mount guards are present. Nothing to unlock." - } - var parts []string - if r.BindGuards > 0 { - parts = append(parts, countLabel(r.BindGuards, "bind mount guard")) - } - if r.ImmutableGuards > 0 { - parts = append(parts, countLabel(r.ImmutableGuards, "immutable flag")) + return guardCheckFacts(r) } - return fmt.Sprintf("Found %s locking the storage. Apply removes them to unlock it.", strings.Join(parts, " and ")) + return guardCheckFacts(r) + " Apply removes them to unlock it." } -// describeGuardApply renders the real-run outcome explanation. +// describeGuardApply renders the real-run outcome: shared facts plus the retry +// instruction phrased as the menu entry the operator would pick again. func describeGuardApply(r orchestrator.GuardCleanupReport) string { if guardApplyClean(r) { - return "Removed the restore mount guards. The storage is unlocked." - } - return "Some guards are still in place (hidden under a live mount). Unmount the datastore and run Cleanup guards again once it is offline." -} - -// countLabel pluralizes "N thing" / "N things". -func countLabel(n int, singular string) string { - if n == 1 { - return fmt.Sprintf("%d %s", n, singular) + return guardApplyFacts(r) } - return fmt.Sprintf("%d %ss", n, singular) + return guardApplyFacts(r) + " Unmount the datastore and run Cleanup guards again once it is offline." } diff --git a/cmd/proxsave/main_modes.go b/cmd/proxsave/main_modes.go index 12fab503..03647035 100644 --- a/cmd/proxsave/main_modes.go +++ b/cmd/proxsave/main_modes.go @@ -204,11 +204,20 @@ func runCleanupGuardsMode(ctx context.Context, args *cli.Args, bootstrap *loggin } logger := logging.New(level, false) - if err := orchestrator.CleanupMountGuards(ctx, logger, args.DryRun); err != nil { + // The REPORT, not the error-only CleanupMountGuards wrapper: exiting 0 with guards + // still holding the storage actively misleads a script gating on the exit code, and + // the read that would have told us was being thrown away. Same seam the dashboard + // uses, so one stub covers both front-ends in tests. + report, err := cleanupGuardsReport(ctx, logger, args.DryRun) + if err != nil { bootstrap.Error("ERROR: %v", err) return types.ExitGenericError.Int(), true } - return types.ExitSuccess.Int(), true + + // The verdict the dashboard shows as CLEAN/FOUND and DONE/PENDING, stated in the + // CLI's voice and then reflected in the exit code. + logCLIGuardVerdict(logger, report, args.DryRun) + return guardCleanupExitCode(report, args.DryRun).Int(), true } func runUpgradeMode(ctx context.Context, args *cli.Args, bootstrap *logging.BootstrapLogger, _ string) (int, bool) { diff --git a/internal/types/exit_codes.go b/internal/types/exit_codes.go index e74eabb3..bdf1b866 100644 --- a/internal/types/exit_codes.go +++ b/internal/types/exit_codes.go @@ -58,6 +58,16 @@ const ( // does not ping a false-green finish for a child that never backed up, and the CLI footer // colors it as a benign skip rather than success or error (F09-03). ExitBackupSkipped ExitCode = 16 + + // ExitGuardsPending - A guard cleanup ran without error but the storage is still + // locked: guard mounts or immutable flags are left behind (typically hidden under a + // live mount), or the remaining count could not be confirmed. Like ExitBackupSkipped + // this is a STATE, not a failure — nothing went wrong, the work simply cannot finish + // until the datastore is offline — so it is kept distinct from ExitGenericError, + // which for this mode means the cleanup itself failed. A script gating on the exit + // code needs the two apart: one is a bug to report, the other is "unmount and retry". + // Also returned by the read-only --dry-run check when guards are found. + ExitGuardsPending ExitCode = 17 ) // String returns a human-readable description of the exit code. @@ -97,6 +107,8 @@ func (e ExitCode) String() string { return "encryption error" case ExitBackupSkipped: return "backup skipped" + case ExitGuardsPending: + return "guards still in place" default: return "unknown error" } From c0cc02a83e10a8c32c5097f389b982eac5390ae6 Mon Sep 17 00:00:00 2001 From: tis24dev Date: Mon, 3 Aug 2026 04:11:04 +0200 Subject: [PATCH 12/50] fix(backup): one statistics recap for both front-ends, and statistics in an unattended run's log MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The recap was written twice — a debug-only log block and a themed graphical block — and the two had drifted apart in three ways. Both now render the rows built by backupStatsRecap; presentation stays with the renderer, content does not. 1. Only the graphical block reported FilesMissing. That is the field the notifications send, so an email could name missing files that the same run's log never mentioned, and the log is what survives to be investigated. The count is now in both. 2. The bundle lines came out in opposite orders, with the graphical block's comment claiming to mirror the log while inverting it. Path first, contents second, matching the plain case where "Archive path" precedes the manifest and checksum describing it. 3. The log block was gated behind DEBUG on the reasoning that "a standard run shows it in the graphical outcome recap instead". That holds only where there IS a graphical recap. logBackupStatistics runs from executeBackupRun, which every backup goes through including the scheduled ones, so an unattended `proxsave --backup` at INFO wrote no statistics anywhere — and a cron log missing the archive path and the file counts is the one place they matter most, because nobody watched the run. INFO now gets compact rows (file counts, archive size, duration, path); the section header and the full detail stay behind DEBUG. One deliberate presentation change: a row the builder marks Warn is rendered warning-coloured whole, where the graphical block used to colour just the "N missing" and "(M failed)" segments inside the Files line. That is the price of both front-ends agreeing on what a row is, and it costs no information — the row is more visible. logCompressionRatio and logBackupArtifactPaths are gone, folded into the builder. TestLogBackupStatisticsDebugGating is renamed and rewritten: its name asserted the Info-emits-nothing behaviour this commit reverses. Verified by mutation: 9 mutations, all caught. --- cmd/proxsave/backup_execution.go | 66 ++++------ cmd/proxsave/backup_execution_test.go | 55 +++++---- cmd/proxsave/backup_stats_recap.go | 100 +++++++++++++++ cmd/proxsave/backup_stats_recap_test.go | 158 ++++++++++++++++++++++++ cmd/proxsave/backup_stream.go | 67 +++------- 5 files changed, 329 insertions(+), 117 deletions(-) create mode 100644 cmd/proxsave/backup_stats_recap.go create mode 100644 cmd/proxsave/backup_stats_recap_test.go diff --git a/cmd/proxsave/backup_execution.go b/cmd/proxsave/backup_execution.go index 7515aaa8..b46315e4 100644 --- a/cmd/proxsave/backup_execution.go +++ b/cmd/proxsave/backup_execution.go @@ -108,40 +108,40 @@ func persistBackupStats(orch *orchestrator.Orchestrator, stats *orchestrator.Bac } } +// logBackupStatistics writes the recap to the run log: the COMPACT rows at Info, the +// full block at Debug. +// +// The block used to be debug-only, on the reasoning that "a standard run shows it in +// the graphical outcome recap instead". That holds only where there IS a graphical +// recap. This function runs from executeBackupRun, which every backup goes through +// including the scheduled ones, so an unattended `proxsave --backup` at Info wrote no +// statistics anywhere — and a cron log with no archive path or file counts is the one +// place they are needed most, since nobody watched the run. func logBackupStatistics(stats *orchestrator.BackupStats) { - // The block is now debug-only; a standard run shows it in the graphical - // outcome recap (buildBackupOutcomePrompt) instead. Guard before the blank - // spacers too, so a standard run shows no orphan blank lines. - if logging.GetDefaultLogger().GetLevel() < types.LogLevelDebug { + if stats == nil { return } + debug := logging.GetDefaultLogger().GetLevel() >= types.LogLevelDebug + + // The header and the blank spacers belong to the full block: at Info the rows join + // the run's other lines instead of opening a section of their own. + if !debug { + for _, line := range backupStatsRecap(stats, true) { + logging.Info("%s", line.Text) + } + return + } + fmt.Println() logging.Debug("=== Backup Statistics ===") - logging.Debug("Files collected: %d", stats.FilesCollected) - if stats.FilesFailed > 0 { - // The PVE/PBS collection summary carries the Files-failed warning now. - logging.Debug("Files failed: %d", stats.FilesFailed) - } - logging.Debug("Directories created: %d", stats.DirsCreated) - logging.Debug("Data collected: %s", formatBytes(stats.BytesCollected)) - logging.Debug("Archive size: %s", formatBytes(stats.ArchiveSize)) - logCompressionRatio(stats) - logging.Debug("Compression used: %s (level %d, mode %s)", stats.Compression, stats.CompressionLevel, stats.CompressionMode) - if stats.RequestedCompression != stats.Compression { - logging.Debug("Requested compression: %s", stats.RequestedCompression) + for _, line := range backupStatsRecap(stats, false) { + logging.Debug("%s", line.Text) } - logging.Debug("Duration: %s", formatDuration(stats.Duration)) - logBackupArtifactPaths(stats) fmt.Println() } -func logCompressionRatio(stats *orchestrator.BackupStats) { - logging.Debug("Compression ratio: %s", compressionRatioText(stats)) -} - -// compressionRatioText renders the compression-ratio value shared by the -// debug-only log block (logCompressionRatio) and the graphical outcome recap -// (appendBackupStatsBlock), so the two never drift. +// compressionRatioText renders the compression-ratio value for the shared recap +// builder (backupStatsRecap), and therefore for both front-ends. func compressionRatioText(stats *orchestrator.BackupStats) string { switch { case stats.CompressionSavingsPercent > 0: @@ -156,22 +156,6 @@ func compressionRatioText(stats *orchestrator.BackupStats) string { } } -func logBackupArtifactPaths(stats *orchestrator.BackupStats) { - if stats.BundleCreated { - logging.Debug("Bundle path: %s", stats.ArchivePath) - logging.Debug("Bundle contents: archive + checksum + metadata") - return - } - - logging.Debug("Archive path: %s", stats.ArchivePath) - if stats.ManifestPath != "" { - logging.Debug("Manifest path: %s", stats.ManifestPath) - } - if stats.Checksum != "" { - logging.Debug("Archive checksum (SHA256): %s", stats.Checksum) - } -} - // consoleStatusGlyph returns a TEXT-presentation glyph (all width 1, terminal-stable) // for the console "Exit status" line, matching the plain checkmarks used everywhere // else in the run output. It deliberately avoids notify.GetStatusEmoji, whose diff --git a/cmd/proxsave/backup_execution_test.go b/cmd/proxsave/backup_execution_test.go index 4ec16fbf..ad228b53 100644 --- a/cmd/proxsave/backup_execution_test.go +++ b/cmd/proxsave/backup_execution_test.go @@ -10,11 +10,12 @@ import ( "github.com/tis24dev/proxsave/internal/types" ) -// TestLogBackupStatisticsDebugGating asserts the "=== Backup Statistics ===" -// block is debug-only: at LogLevelInfo logBackupStatistics emits nothing, and at -// LogLevelDebug it emits the full block. The block moved to the graphical outcome -// recap (buildBackupOutcomePrompt) for standard runs. -func TestLogBackupStatisticsDebugGating(t *testing.T) { +// TestLogBackupStatisticsLevelSplit asserts what each log level gets. This test used +// to be TestLogBackupStatisticsDebugGating and asserted that Info emits NOTHING — the +// behaviour a cron run inherited, which left an unattended backup with no statistics +// anywhere. Info now gets the compact rows; only the "=== Backup Statistics ===" header +// and the full detail stay behind Debug. +func TestLogBackupStatisticsLevelSplit(t *testing.T) { prevLogger := logging.GetDefaultLogger() t.Cleanup(func() { logging.SetDefaultLogger(prevLogger) }) @@ -24,26 +25,34 @@ func TestLogBackupStatisticsDebugGating(t *testing.T) { ArchivePath: "/var/backup/proxsave.tar.zst", } - // At Info level the block (and its blank spacers) is skipped entirely. - infoBuf := &bytes.Buffer{} - infoLogger := logging.New(types.LogLevelInfo, false) - infoLogger.SetOutput(infoBuf) - logging.SetDefaultLogger(infoLogger) - logBackupStatistics(stats) - if strings.Contains(infoBuf.String(), "=== Backup Statistics ===") { - t.Fatalf("stats block must be absent at Info level:\n%s", infoBuf.String()) + capture := func(level types.LogLevel) string { + buf := &bytes.Buffer{} + logger := logging.New(level, false) + logger.SetOutput(buf) + logging.SetDefaultLogger(logger) + logBackupStatistics(stats) + return buf.String() } - // At Debug level the full block is emitted. - debugBuf := &bytes.Buffer{} - debugLogger := logging.New(types.LogLevelDebug, false) - debugLogger.SetOutput(debugBuf) - logging.SetDefaultLogger(debugLogger) - logBackupStatistics(stats) - if !strings.Contains(debugBuf.String(), "=== Backup Statistics ===") { - t.Fatalf("stats block must be present at Debug level:\n%s", debugBuf.String()) + // Info: the compact rows, no section header. This is what a cron log gets. + info := capture(types.LogLevelInfo) + if strings.Contains(info, "=== Backup Statistics ===") { + t.Fatalf("the section header belongs to the full block:\n%s", info) } - if !strings.Contains(debugBuf.String(), "Files collected: 42") { - t.Fatalf("stats block content missing at Debug level:\n%s", debugBuf.String()) + for _, want := range []string{"Files: 42 collected", "/var/backup/proxsave.tar.zst"} { + if !strings.Contains(info, want) { + t.Fatalf("an unattended run must still log %q:\n%s", want, info) + } + } + if strings.Contains(info, "Directories created") { + t.Fatalf("the compact rows must stay compact:\n%s", info) + } + + // Debug: the full block, header included. + debug := capture(types.LogLevelDebug) + for _, want := range []string{"=== Backup Statistics ===", "Files: 42 collected", "Directories created: 7"} { + if !strings.Contains(debug, want) { + t.Fatalf("the full block must contain %q:\n%s", want, debug) + } } } diff --git a/cmd/proxsave/backup_stats_recap.go b/cmd/proxsave/backup_stats_recap.go new file mode 100644 index 00000000..882a78fa --- /dev/null +++ b/cmd/proxsave/backup_stats_recap.go @@ -0,0 +1,100 @@ +package main + +import ( + "fmt" + + "github.com/tis24dev/proxsave/internal/orchestrator" +) + +// The backup-statistics recap, single-sourced for BOTH front-ends. +// +// It was written twice — a debug-only log block and a themed graphical block — and the +// two had drifted: only the graphical one reported FilesMissing (the field the +// notifications send, so an email could name missing files the same run's log never +// did), and the bundle lines came out in opposite orders. Building the ROWS here and +// leaving each front-end to present them is what stops that: adding a row reaches both, +// and neither can quietly stop showing one. +// +// Presentation stays with the renderer. The graphical block themes the rows; the log +// block writes them through the logger. Neither decides WHAT the recap says. + +// backupStatLine is one recap row. Warn marks a row reporting something the operator +// should look at (missing or failed files) — the graphical block renders those in the +// warning colour. The log block ignores it: a log line carries its level, not a colour, +// and the whole block already sits at one level. +type backupStatLine struct { + Text string + Warn bool +} + +// backupStatsRecap builds the recap rows. +// +// compact keeps only what answers "did it work, and where is the archive": the file +// counts, the archive size, how long it took, and the path. It exists because the full +// block is DEBUG-only while logBackupStatistics runs on every backup including the +// unattended ones, which left a cron run at INFO with no statistics at all and no +// graphical recap to stand in for them. +func backupStatsRecap(st *orchestrator.BackupStats, compact bool) []backupStatLine { + if st == nil { + return nil + } + + // Files first, and always: the collected/missing/failed triple is the one row that + // says whether the backup is complete. "missing" is st.FilesMissing, the SAME field + // the notifications report — a mail saying "5 missing" that the log never mentions + // cannot be reconciled by whoever is investigating. + files := fmt.Sprintf("Files: %d collected - %d missing", st.FilesCollected, st.FilesMissing) + if st.FilesFailed > 0 { + files += fmt.Sprintf(" (%d failed)", st.FilesFailed) + } + lines := []backupStatLine{{Text: files, Warn: st.FilesMissing > 0 || st.FilesFailed > 0}} + + if !compact { + lines = append(lines, + backupStatLine{Text: fmt.Sprintf("Directories created: %d", st.DirsCreated)}, + backupStatLine{Text: "Data collected: " + formatBytes(st.BytesCollected)}, + ) + } + lines = append(lines, backupStatLine{Text: "Archive size: " + formatBytes(st.ArchiveSize)}) + if !compact { + lines = append(lines, + backupStatLine{Text: "Compression ratio: " + compressionRatioText(st)}, + backupStatLine{Text: fmt.Sprintf("Compression used: %s (level %d, mode %s)", st.Compression, st.CompressionLevel, st.CompressionMode)}, + ) + if st.RequestedCompression != st.Compression { + lines = append(lines, backupStatLine{Text: fmt.Sprintf("Requested compression: %s", st.RequestedCompression)}) + } + } + lines = append(lines, backupStatLine{Text: "Duration: " + formatDuration(st.Duration)}) + + return append(lines, backupArtifactLines(st, compact)...) +} + +// backupArtifactLines names where the run's output landed. The bundle case reports the +// PATH first and its contents second, matching the plain case where "Archive path" +// precedes the manifest and checksum that describe it — the graphical block used to +// invert the pair while its own comment claimed to mirror the log. +// +// compact keeps only the path itself: the manifest and checksum are derivable from it +// and belong to the full block. +func backupArtifactLines(st *orchestrator.BackupStats, compact bool) []backupStatLine { + if st.BundleCreated { + lines := []backupStatLine{{Text: "Bundle path: " + st.ArchivePath}} + if compact { + return lines + } + return append(lines, backupStatLine{Text: "Bundle contents: archive + checksum + metadata"}) + } + + lines := []backupStatLine{{Text: "Archive path: " + st.ArchivePath}} + if compact { + return lines + } + if st.ManifestPath != "" { + lines = append(lines, backupStatLine{Text: "Manifest path: " + st.ManifestPath}) + } + if st.Checksum != "" { + lines = append(lines, backupStatLine{Text: "Archive checksum (SHA256): " + st.Checksum}) + } + return lines +} diff --git a/cmd/proxsave/backup_stats_recap_test.go b/cmd/proxsave/backup_stats_recap_test.go new file mode 100644 index 00000000..ddb62e6f --- /dev/null +++ b/cmd/proxsave/backup_stats_recap_test.go @@ -0,0 +1,158 @@ +package main + +import ( + "strings" + "testing" + "time" + + "github.com/tis24dev/proxsave/internal/orchestrator" +) + +func recapText(lines []backupStatLine) string { + parts := make([]string, 0, len(lines)) + for _, l := range lines { + parts = append(parts, l.Text) + } + return strings.Join(parts, "\n") +} + +func fullStats() *orchestrator.BackupStats { + return &orchestrator.BackupStats{ + FilesCollected: 40, + FilesMissing: 5, + FilesFailed: 3, + DirsCreated: 7, + BytesCollected: 8192, + ArchiveSize: 4096, + Compression: "zstd", + CompressionLevel: 3, + CompressionMode: "standard", + RequestedCompression: "zstd", + Duration: 90 * time.Second, + ArchivePath: "/var/backup/proxsave.tar.zst", + ManifestPath: "/var/backup/proxsave.manifest", + Checksum: "abc123", + } +} + +// TestBackupStatsRecapReportsMissingFiles is the regression proper: FilesMissing is the +// field the notifications send, and only the graphical block reported it. An email +// saying "5 missing" that the same run's log never mentions cannot be reconciled by +// whoever is investigating, and the log is what survives. +func TestBackupStatsRecapReportsMissingFiles(t *testing.T) { + for _, compact := range []bool{false, true} { + out := recapText(backupStatsRecap(fullStats(), compact)) + if !strings.Contains(out, "Files: 40 collected - 5 missing") { + t.Errorf("compact=%v: the missing count must be reported:\n%s", compact, out) + } + if !strings.Contains(out, "(3 failed)") { + t.Errorf("compact=%v: the failed count must be reported:\n%s", compact, out) + } + } +} + +// TestBackupStatsRecapMarksTheFilesRowForAttention: the row carries its own severity so +// a renderer does not have to re-derive it from the numbers and get it wrong. Zero +// missing and zero failed is an ordinary row. +func TestBackupStatsRecapMarksTheFilesRowForAttention(t *testing.T) { + cases := []struct { + name string + missing int + failed int + wantWarn bool + }{ + {"complete", 0, 0, false}, + {"files missing", 5, 0, true}, + {"files failed", 0, 3, true}, + {"both", 5, 3, true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + st := fullStats() + st.FilesMissing, st.FilesFailed = tc.missing, tc.failed + lines := backupStatsRecap(st, false) + if len(lines) == 0 { + t.Fatal("the recap must not be empty") + } + if !strings.HasPrefix(lines[0].Text, "Files:") { + t.Fatalf("the files row must come first, got %q", lines[0].Text) + } + if lines[0].Warn != tc.wantWarn { + t.Fatalf("Warn = %v, want %v for %q", lines[0].Warn, tc.wantWarn, lines[0].Text) + } + }) + } +} + +// TestBackupStatsRecapPutsTheBundlePathBeforeItsContents: the two front-ends emitted +// this pair in opposite orders, and the graphical one claimed in a comment to mirror +// the log while inverting it. Path first matches the plain case, where "Archive path" +// precedes the manifest and checksum that describe it. +func TestBackupStatsRecapPutsTheBundlePathBeforeItsContents(t *testing.T) { + st := fullStats() + st.BundleCreated = true + out := recapText(backupStatsRecap(st, false)) + + path, contents := strings.Index(out, "Bundle path:"), strings.Index(out, "Bundle contents:") + if path < 0 || contents < 0 { + t.Fatalf("both bundle lines must be present:\n%s", out) + } + if path > contents { + t.Fatalf("the bundle path must precede its contents (path at %d, contents at %d):\n%s", path, contents, out) + } + // A bundle run names no bare archive: the bundle IS the artifact. + if strings.Contains(out, "Archive path:") { + t.Fatalf("a bundle run must not also report an archive path:\n%s", out) + } +} + +// TestBackupStatsRecapCompactAnswersTheCronQuestions: the compact rows exist for the +// unattended run, and are worth nothing if they omit what a post-mortem starts from — +// whether the backup is complete, how big it is, how long it took, where it landed. +func TestBackupStatsRecapCompactAnswersTheCronQuestions(t *testing.T) { + out := recapText(backupStatsRecap(fullStats(), true)) + for _, want := range []string{"Files: 40 collected", "Archive size:", "Duration:", "Archive path: /var/backup/proxsave.tar.zst"} { + if !strings.Contains(out, want) { + t.Errorf("the compact recap must answer %q:\n%s", want, out) + } + } + // And it must stay compact, or it is just the full block under another name. + for _, unwanted := range []string{"Directories created", "Data collected", "Compression", "Manifest path", "checksum"} { + if strings.Contains(out, unwanted) { + t.Errorf("the compact recap must drop %q:\n%s", unwanted, out) + } + } +} + +// TestBackupStatsRecapFullKeepsEveryDetail guards the other direction: the compact +// split must not quietly cost the full block a row. +func TestBackupStatsRecapFullKeepsEveryDetail(t *testing.T) { + st := fullStats() + st.RequestedCompression = "xz" // differs from Compression, so the extra row appears + out := recapText(backupStatsRecap(st, false)) + for _, want := range []string{ + "Files: 40 collected - 5 missing (3 failed)", + "Directories created: 7", + "Data collected: 8.0 KiB", + "Archive size: 4.0 KiB", + "Compression ratio: 50.0%", + "Compression used: zstd (level 3, mode standard)", + "Requested compression: xz", + "Duration:", + "Archive path: /var/backup/proxsave.tar.zst", + "Manifest path: /var/backup/proxsave.manifest", + "Archive checksum (SHA256): abc123", + } { + if !strings.Contains(out, want) { + t.Errorf("the full recap must contain %q:\n%s", want, out) + } + } +} + +// TestBackupStatsRecapToleratesNilStats: the graphical recap is also built for runs that +// died before any stats existed, and the builder is reached through a nil-able pointer. +func TestBackupStatsRecapToleratesNilStats(t *testing.T) { + if lines := backupStatsRecap(nil, false); lines != nil { + t.Fatalf("nil stats must yield no rows, got %v", lines) + } +} diff --git a/cmd/proxsave/backup_stream.go b/cmd/proxsave/backup_stream.go index ca92f18f..7c1ef2d2 100644 --- a/cmd/proxsave/backup_stream.go +++ b/cmd/proxsave/backup_stream.go @@ -278,61 +278,22 @@ func buildBackupOutcomePrompt(res backupModeResult) string { } // appendBackupStatsBlock renders the backup-statistics block into the graphical -// outcome recap. Its first line is the enriched "Files: N collected - K missing -// (M failed)" moved down from the upper recap; the remaining lines mirror the -// debug-only log block in logBackupStatistics (same lines, conditionals and -// formatters - formatBytes/formatDuration and the shared compressionRatioText), -// just THEME-styled instead of logged. +// outcome recap: the shared rows from backupStatsRecap, THEME-styled. The graphical +// front-end always shows the full block — it is the only recap a dashboard run gets, +// and it has the room. +// +// A row the builder marks Warn is rendered warning-coloured whole. It used to colour +// just the "N missing" and "(M failed)" segments inside the Files line; whole-row is +// the price of the two front-ends agreeing on what a row IS, and it loses nothing — +// the row is more visible, not less. func appendBackupStatsBlock(b *strings.Builder, st *orchestrator.BackupStats) { - // Files: N collected - K missing (M failed) - moved down from the upper recap. - // "missing" reuses st.FilesMissing (the field the notifications report), always - // shown (yellow when >0); the failed count only when non-zero. - b.WriteString("\n") - b.WriteString(theme.Text.Render(fmt.Sprintf("Files: %d collected - ", st.FilesCollected))) - missingStyle := theme.Text - if st.FilesMissing > 0 { - missingStyle = theme.WarningText - } - b.WriteString(missingStyle.Render(fmt.Sprintf("%d missing", st.FilesMissing))) - if st.FilesFailed > 0 { - b.WriteString(theme.WarningText.Render(fmt.Sprintf(" (%d failed)", st.FilesFailed))) - } - b.WriteString("\n") - b.WriteString(theme.Text.Render(fmt.Sprintf("Directories created: %d", st.DirsCreated))) - b.WriteString("\n") - b.WriteString(theme.Text.Render("Data collected: " + formatBytes(st.BytesCollected))) - b.WriteString("\n") - b.WriteString(theme.Text.Render("Archive size: " + formatBytes(st.ArchiveSize))) - b.WriteString("\n") - b.WriteString(theme.Text.Render("Compression ratio: " + compressionRatioText(st))) - b.WriteString("\n") - b.WriteString(theme.Text.Render(fmt.Sprintf("Compression used: %s (level %d, mode %s)", st.Compression, st.CompressionLevel, st.CompressionMode))) - if st.RequestedCompression != st.Compression { - b.WriteString("\n") - b.WriteString(theme.Text.Render(fmt.Sprintf("Requested compression: %s", st.RequestedCompression))) - } - b.WriteString("\n") - b.WriteString(theme.Text.Render("Duration: " + formatDuration(st.Duration))) - - // Mirror logBackupArtifactPaths (bundle case shown contents-then-path). A base - // (non-bundle) run has no "Bundle contents" line: it shows "Archive path" plus - // the manifest/checksum, so this line adapts to the configured mode. - if st.BundleCreated { - b.WriteString("\n") - b.WriteString(theme.Text.Render("Bundle contents: archive + checksum + metadata")) - b.WriteString("\n") - b.WriteString(theme.Text.Render("Bundle path: " + st.ArchivePath)) - return - } - b.WriteString("\n") - b.WriteString(theme.Text.Render("Archive path: " + st.ArchivePath)) - if st.ManifestPath != "" { - b.WriteString("\n") - b.WriteString(theme.Text.Render("Manifest path: " + st.ManifestPath)) - } - if st.Checksum != "" { + for _, line := range backupStatsRecap(st, false) { + style := theme.Text + if line.Warn { + style = theme.WarningText + } b.WriteString("\n") - b.WriteString(theme.Text.Render("Archive checksum (SHA256): " + st.Checksum)) + b.WriteString(style.Render(line.Text)) } } From 9a60f44239f15ae4636781dc5bf68c52f2159f2f Mon Sep 17 00:00:00 2001 From: tis24dev Date: Mon, 3 Aug 2026 12:15:25 +0200 Subject: [PATCH 13/50] fix(backup): render the statistics recap once per screen again c0cc02a lifted the Debug gate on logBackupStatistics so an unattended run would still log statistics. It read that gate as a verbosity choice. It was not: the gate and the graphical stats block arrived together in 3ed0716 as two halves of one design, where exactly one side renders the recap. Lifting it put the compact rows into the dashboard viewport, underneath the outcome block that was already showing the same four rows. The engine now skips its own recap when the caller says the screen renders it, via backupModeOptions.outcomeRendersRecap. The field states a fact about the RUN, not about which front-end is driving, so the engine stays as ignorant of its renderer as backupStatsRecap already is - the same split cleanup_guards_verdict.go documents. It is set on the streamed path's own clone of the options and nowhere else. The trap here is real: runBackupStreamed also runs the SAME steps with the un-cloned options when the dashboard handoff has vanished, and that run shows no outcome block at all. Setting the flag one line earlier would have stripped the statistics from precisely the run that has no other copy of them. A test pins the assignment below the fallback. This also closes a case the original gate never covered. At Debug the full block reached the viewport too, because capturing the console swaps the writer and not the level, so a dashboard --support run has been showing the block twice since 3ed0716. Reverting c0cc02a would have left that half in place. Two comments corrected. backup_stream.go said "the log block is now debug-only", which c0cc02a had made false. backup_execution.go named executeBackupRun, a function that does not exist - the caller is runConfiguredBackup. One more correction, to c0cc02a's own reasoning: the recap never reaches the run log FILE at any level. runConfiguredBackup calls this after the orchestrator has closed that file, so the rows go to stdout only. The benefit is real but different from what was claimed - the daemon runs the backup as a child whose stdout feeds journald and the bounded tail POSTed to Healthchecks as the failure diagnostic. That is why the repair suppresses on the caller's say-so rather than deleting the Info arm. Verified by mutation: 6 mutations, all caught, including the shared-options trap and the Info-only half-fix. --- cmd/proxsave/backup_execution.go | 32 ++++++++----- cmd/proxsave/backup_execution_test.go | 69 ++++++++++++++++++++++++++- cmd/proxsave/backup_mode.go | 11 +++++ cmd/proxsave/backup_stream.go | 18 +++++-- 4 files changed, 115 insertions(+), 15 deletions(-) diff --git a/cmd/proxsave/backup_execution.go b/cmd/proxsave/backup_execution.go index b46315e4..4d9f134a 100644 --- a/cmd/proxsave/backup_execution.go +++ b/cmd/proxsave/backup_execution.go @@ -43,7 +43,7 @@ func runConfiguredBackup(opts backupModeOptions, orch *orchestrator.Orchestrator backupDone(nil) persistBackupStats(orch, stats) - logBackupStatistics(stats) + logBackupStatistics(stats, opts.outcomeRendersRecap) logging.Info("✓ Backup completed") logServerIdentityValues(opts.serverIDValue, opts.serverMACValue) logMonitoringPortalLink(stats) @@ -108,17 +108,27 @@ func persistBackupStats(orch *orchestrator.Orchestrator, stats *orchestrator.Bac } } -// logBackupStatistics writes the recap to the run log: the COMPACT rows at Info, the -// full block at Debug. +// logBackupStatistics writes the recap to the console: the COMPACT rows at Info, the +// full block at Debug. skip suppresses BOTH, for a run whose screen renders the recap +// itself (backupModeOptions.outcomeRendersRecap). // -// The block used to be debug-only, on the reasoning that "a standard run shows it in -// the graphical outcome recap instead". That holds only where there IS a graphical -// recap. This function runs from executeBackupRun, which every backup goes through -// including the scheduled ones, so an unattended `proxsave --backup` at Info wrote no -// statistics anywhere — and a cron log with no archive path or file counts is the one -// place they are needed most, since nobody watched the run. -func logBackupStatistics(stats *orchestrator.BackupStats) { - if stats == nil { +// The block was debug-only before c0cc02a, and that gate WAS the de-duplication: the +// graphical stats block and the Debug gate arrived together in 3ed0716, two halves of +// one design where exactly one side renders the recap. Reading the gate as a verbosity +// choice and lifting it for Info put the compact rows into the dashboard viewport +// underneath the outcome block that already showed them. Suppressing here on the +// caller's say-so restores that invariant, and closes the case the original gate never +// covered: at Debug the full block reached the viewport too, because capturing the +// console swaps the writer and not the level (internal/logging/capture.go, SwapOutput). +// +// Where the rows go on an unattended run, corrected: NOT into the run log file. This +// runs from runConfiguredBackup, after the orchestrator has closed that file +// (FinalizeAndCloseLog), so the recap reaches stdout only. It still matters there: the +// daemon runs the backup as a child whose stdout goes to journald AND into the bounded +// tail it POSTs to Healthchecks as the failure diagnostic (daemon.go, buildBackupCmd). +// Deleting the Info arm would take that payload away. +func logBackupStatistics(stats *orchestrator.BackupStats, skip bool) { + if stats == nil || skip { return } debug := logging.GetDefaultLogger().GetLevel() >= types.LogLevelDebug diff --git a/cmd/proxsave/backup_execution_test.go b/cmd/proxsave/backup_execution_test.go index ad228b53..0fafc37b 100644 --- a/cmd/proxsave/backup_execution_test.go +++ b/cmd/proxsave/backup_execution_test.go @@ -2,6 +2,7 @@ package main import ( "bytes" + "os" "strings" "testing" @@ -30,7 +31,7 @@ func TestLogBackupStatisticsLevelSplit(t *testing.T) { logger := logging.New(level, false) logger.SetOutput(buf) logging.SetDefaultLogger(logger) - logBackupStatistics(stats) + logBackupStatistics(stats, false) return buf.String() } @@ -56,3 +57,69 @@ func TestLogBackupStatisticsLevelSplit(t *testing.T) { } } } + +// TestLogBackupStatisticsSkippedWhenTheOutcomeRendersIt: the dashboard screen renders +// the recap itself in its outcome block, so the engine must stay silent there — at +// EVERY level. c0cc02a suppressed only the Debug arm's predecessor and let the compact +// rows into the run viewport, where they landed underneath the same four rows the +// outcome block was already showing. Debug was worse and older: capturing the console +// swaps the writer, not the level, so the full block reached the viewport too. +func TestLogBackupStatisticsSkippedWhenTheOutcomeRendersIt(t *testing.T) { + prevLogger := logging.GetDefaultLogger() + t.Cleanup(func() { logging.SetDefaultLogger(prevLogger) }) + + stats := &orchestrator.BackupStats{ + FilesCollected: 42, + DirsCreated: 7, + ArchivePath: "/var/backup/proxsave.tar.zst", + } + + for _, level := range []types.LogLevel{types.LogLevelInfo, types.LogLevelDebug} { + buf := &bytes.Buffer{} + logger := logging.New(level, false) + logger.SetOutput(buf) + logging.SetDefaultLogger(logger) + + logBackupStatistics(stats, true) + + if out := buf.String(); strings.TrimSpace(out) != "" { + t.Errorf("level %v: the engine must log nothing when the outcome renders the recap, got %q", level, out) + } + } +} + +// TestStreamedBackupSuppressesTheLoggedRecapOnlyOnTheViewportPath is the trap this +// repair had to avoid. runBackupStreamed has TWO ways to run the same steps: with a +// live session (viewport + outcome block, so the logged recap would be a duplicate) and +// without one, when the dashboard handoff vanished and the run continues plain. The +// second shows no outcome at all, so it must keep the logged recap — putting the flag +// on the shared options instead of on the viewport clone would have silently stripped +// the statistics from exactly the run that has no other copy of them. +func TestStreamedBackupSuppressesTheLoggedRecapOnlyOnTheViewportPath(t *testing.T) { + src, err := os.ReadFile("backup_stream.go") + if err != nil { + t.Fatalf("read backup_stream.go: %v", err) + } + body := string(src) + + // Count ASSIGNMENTS, not mentions: the field is named in comments here too, and a + // test that counted the bare name would fail on documentation. + const set = "stepOpts.outcomeRendersRecap = true" + if n := strings.Count(body, "outcomeRendersRecap = "); n != 1 { + t.Fatalf("the flag must be assigned exactly once in this file, found %d assignments", n) + } + setIdx := strings.Index(body, set) + if setIdx < 0 { + t.Fatalf("the flag must be set on the stepOpts CLONE, not on opts") + } + // The fallback runs the steps with the un-cloned opts and returns before the + // viewport exists; the clone is created after it. If the set ever moves above the + // fallback it would start applying to a run that renders no outcome. + fallbackIdx := strings.Index(body, "res := backupStreamSteps(opts)") + if fallbackIdx < 0 { + t.Fatal("the no-session fallback call site moved; re-check which options it passes") + } + if setIdx < fallbackIdx { + t.Fatalf("the flag is set before the no-session fallback (set=%d fallback=%d): that run has no outcome block and must keep the logged recap", setIdx, fallbackIdx) + } +} diff --git a/cmd/proxsave/backup_mode.go b/cmd/proxsave/backup_mode.go index 421605fb..0e3f2df9 100644 --- a/cmd/proxsave/backup_mode.go +++ b/cmd/proxsave/backup_mode.go @@ -39,6 +39,17 @@ type backupModeOptions struct { // deferred sender. support bool supportMeta support.Meta + // outcomeRendersRecap says that THIS run ends on a screen that renders the + // backup-statistics recap itself, so the engine must not also log it. Only the + // streamed path with a live viewport sets it (runBackupStreamed), and only on its + // own clone of these options: the plain path, the daemon, cron, and the streamed + // path's no-session fallback all leave it zero and keep the logged recap, which is + // the only one they get. + // + // It states a fact about the RUN, not about which front-end is driving. The engine + // stays ignorant of its renderer, the same way backupStatsRecap builds rows without + // knowing who will present them. + outcomeRendersRecap bool } type backupModeResult struct { diff --git a/cmd/proxsave/backup_stream.go b/cmd/proxsave/backup_stream.go index 7c1ef2d2..4cd6ec6b 100644 --- a/cmd/proxsave/backup_stream.go +++ b/cmd/proxsave/backup_stream.go @@ -164,8 +164,17 @@ func runBackupStreamed(opts backupModeOptions) backupModeResult { defer captureRunOutput(opts.bootstrap, emit)() // Thread taskCtx so an Esc cancel propagates into the running backup. + // + // outcomeRendersRecap is set HERE and nowhere else: this is the only path + // that ends on buildBackupOutcomePrompt, so it is the only one where the + // engine logging the recap would put it on screen twice. The no-session + // fallback above runs the SAME steps with the un-cloned opts and shows no + // outcome at all, so it keeps the logged recap - setting the flag on opts + // instead of on this clone would silently strip the statistics from exactly + // the run that has no other recap. stepOpts := opts stepOpts.ctx = taskCtx + stepOpts.outcomeRendersRecap = true res = backupStreamSteps(stepOpts) runStreamedEndOfRunActions(taskCtx, opts, &res) return buildBackupOutcomePrompt(res), nil @@ -221,7 +230,9 @@ func buildBackupOutcomePrompt(res backupModeResult) string { b.WriteString(renderBackupBanner(sev)) if st := res.supportStats; st != nil { - // The backup-statistics block (headerless); the log block is now debug-only. + // The backup-statistics block (headerless). This screen is the ONLY recap of + // the run: the engine skips its own logged recap for this path, so the rows + // below appear exactly once (backupModeOptions.outcomeRendersRecap). b.WriteString("\n") appendBackupStatsBlock(&b, st) @@ -279,8 +290,9 @@ func buildBackupOutcomePrompt(res backupModeResult) string { // appendBackupStatsBlock renders the backup-statistics block into the graphical // outcome recap: the shared rows from backupStatsRecap, THEME-styled. The graphical -// front-end always shows the full block — it is the only recap a dashboard run gets, -// and it has the room. +// front-end always shows the full block, and it really is the only recap a dashboard +// run gets: the engine's own logged recap is suppressed for this path, at every log +// level (see logBackupStatistics). // // A row the builder marks Warn is rendered warning-coloured whole. It used to colour // just the "N missing" and "(M failed)" segments inside the Files line; whole-row is From 018bd021bd424029ef0d9ae90ed3954e397b2ff4 Mon Sep 17 00:00:00 2001 From: tis24dev Date: Mon, 3 Aug 2026 14:21:50 +0200 Subject: [PATCH 14/50] docs: correct the config-upgrade backup path and the "removed variables" claim Two errors in the --upgrade-config process list, both found by reading the code beside it. The backup filename was documented as `backup.env.bak-YYYYMMDD-HHMMSS`. The code writes `.backup.` (internal/config/upgrade.go:101-102) - different separator, different timestamp format. An operator following the docs looks for a file that does not exist and concludes no backup was taken, when one is sitting next to the config. The documented shape is not invented: `.bak-` is what --newkey uses for the recipient file (internal/orchestrator/encryption.go:406) and what the restore path uses for replaced files, so the docs had borrowed a sibling mechanism's convention. Those other mentions, in ENCRYPTION.md, are correct and are left alone. "Reports added/removed variables" was worse, because it describes the opposite of what happens to configuration the operator cares about. Nothing is removed. Keys present in the config but not in the template are ExtraKeys, and keys differing from a template key only by case are CaseConflictKeys; both are preserved in place, with the original value and casing (internal/config/upgrade.go, UpgradeResult). A reader with custom keys had reason to expect an upgrade to drop them. Also documents the automatic rollback: an upgraded config that fails validation is replaced by the backup and the error is reported (internal/config/upgrade.go:123-125). That is the reassurance the step list was missing, and it is what makes the backup path worth documenting correctly in the first place. --- docs/CLI_REFERENCE.md | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/docs/CLI_REFERENCE.md b/docs/CLI_REFERENCE.md index fde78aa8..ed1ecd13 100644 --- a/docs/CLI_REFERENCE.md +++ b/docs/CLI_REFERENCE.md @@ -179,9 +179,18 @@ proxsave --upgrade-config-dry-run 1. Reads current `configs/backup.env` 2. Extracts embedded template from binary 3. Merges your values with new template -4. Backs up old config (`backup.env.bak-YYYYMMDD-HHMMSS`) +4. Backs up old config (`backup.env.backup.YYYYMMDD_HHMMSS`, next to the config file) 5. Writes updated configuration -6. Reports added/removed variables +6. Reports added keys, preserved values, and any merge warnings + +**Nothing is ever removed from your configuration.** Keys you set that are not in the +template (including keys that differ from a template key only by upper/lower case) are +preserved in place with their original value and casing, and are reported as such. The +upgrade only adds keys the template has and your config lacks. + +If the merged configuration fails validation, the backup from step 4 is restored +automatically and the command reports the error, so a failed upgrade leaves your +configuration as it was. > **Keep `backup.env` a regular file.** The config upgrade (`--upgrade`, `--upgrade-config`) writes the new configuration atomically (temp file + rename), so if `configs/backup.env` is a **symlink** it is replaced by a regular file and the symlink target is left unchanged. For a centrally managed configuration, deploy a regular `backup.env` (for example copied or templated by your config-management tool) instead of symlinking it. From f1b1f775d62e26662d0d18c3b3ae6114d9df0a06 Mon Sep 17 00:00:00 2001 From: tis24dev Date: Mon, 3 Aug 2026 15:36:05 +0200 Subject: [PATCH 15/50] refactor(daemon): one classifier for the restart-verify verdict, shared by both front-ends Three surfaces classified the same RestartVerifyResult with three hand-copied switches: the CLI upgrade footer, the upgrade bootstrap log, and the dashboard result screen. The first two classify the SAME value on the SAME --upgrade run, so a change to a guard or to the branch order in one could leave the other two silently behind. classifyRestartVerify is now the single verdict; each surface keeps its own wording, its own output channel, and its own version source (the footer names the version just installed, the dashboard names the version the daemon reports -- different facts that must not be merged). The branch order is the contract, not an accident, and it now has a test: the flags are not mutually exclusive, and testing TimedOut before the success conjunction is what stops a restart that ran out of budget from being reported as "now aligned". Two behaviour changes ride along, both of them cases where the two front-ends disagreed about the same engine outcome: - A failed restart is now a WARNING on the dashboard too, not a hard error. The CLI footer has always painted it yellow; the dashboard painted it red. Yellow is the correct side: the new binary is already installed and the daemon merely still runs the old one, and daemonStatusStyle -- the repo's other daemon verdict -- has no red state at all. Severity is now single-sourced, so the two cannot drift apart again. - The poll-only install verdict no longer reports a live daemon as "not running". When the process is alive but the /proc alignment probe never returned a verdict, alignment is UNKNOWN; saying "not running" was wrong about the process and silent about what actually could not be established. That case gets its own arm. "not running" now only appears when it is a measured fact. Also corrects three comments that had drifted into stating the opposite of the code: the poll-only verify DOES set FreshInfo (with a different meaning from the restart path), and installVerifyVerdict does NOT give the same verdict as --daemon-status, whose green arm requires a fresh heartbeat and ignores alignment -- the two answer different questions. Tests: first coverage of the bootstrap log surface, which had none despite being what an unattended upgrade leaves on disk; first coverage of the config-unreadable deferral, which was indistinguishable from the backup deferral because both lines merely contain "deferred"; and pins for the branch order, the severity agreement between the front-ends, and the poll-only separation. Verified by mutation: 11 one-line production edits, all 11 killed by a named test. --- cmd/proxsave/daemon_restart_verdict.go | 94 ++++++++++ cmd/proxsave/daemon_restart_verdict_test.go | 174 +++++++++++++++++++ cmd/proxsave/daemon_restart_verify.go | 51 +++--- cmd/proxsave/daemon_restart_verify_test.go | 99 ++++++++++- cmd/proxsave/daemon_setup.go | 38 +++- cmd/proxsave/dashboard.go | 45 +++-- cmd/proxsave/install_finalize_stream_test.go | 4 +- cmd/proxsave/install_outcome.go | 6 +- cmd/proxsave/upgrade.go | 22 ++- 9 files changed, 470 insertions(+), 63 deletions(-) create mode 100644 cmd/proxsave/daemon_restart_verdict.go create mode 100644 cmd/proxsave/daemon_restart_verdict_test.go diff --git a/cmd/proxsave/daemon_restart_verdict.go b/cmd/proxsave/daemon_restart_verdict.go new file mode 100644 index 00000000..64ebf6e5 --- /dev/null +++ b/cmd/proxsave/daemon_restart_verdict.go @@ -0,0 +1,94 @@ +package main + +import "github.com/tis24dev/proxsave/internal/orchestrator" + +// restartVerifyOutcome is the SINGLE classification of a restart+verify result +// (restartAndVerifyDaemon). Three surfaces render it -- the CLI upgrade footer +// (summarizeRestartVerify), the upgrade bootstrap log (logUpgradeDaemonRestart) and the +// dashboard result screen (restartVerifyStatus) -- and the first two classify the SAME +// value on the SAME --upgrade run (upgrade.go). They used to re-derive the verdict from +// three hand-copied switches, so a change to the order or to a guard in one could leave +// the other two silently behind. +// +// It carries NO text and NO data. Each surface keeps its own wording, its own output +// channel (the log's stdout-vs-stderr split is Println vs Warning, not a colour) and its +// own version SOURCE: the footer names the version the upgrade just INSTALLED, while the +// dashboard names the version the running daemon REPORTS. Those are different facts and +// must not be merged. Only the VERDICT and its severity are shared. +// +// It is defined over a RESTART result. The poll-only verifyDaemonAligned result is +// classified separately by installVerifyVerdict -- see the note there for why the two +// cannot share this. +type restartVerifyOutcome int + +const ( + // restartVerifyError: the restart call itself failed, so nothing was restarted. + // rv.Err is non-nil on this arm and ONLY on this arm -- two surfaces dereference it + // with no nil check, so widening this guard panics them. + restartVerifyError restartVerifyOutcome = iota + // restartVerifyDeferredConfig: the config was unreadable, so the REAL backup lock path + // is unknown; the restart was deferred fail-closed rather than risk killing a backup on + // a custom LOCK_PATH (F11-08). + restartVerifyDeferredConfig + // restartVerifyDeferredBackup: a backup still held the lock when the bounded wait + // elapsed; the restart was deferred rather than killing it. + restartVerifyDeferredBackup + // restartVerifyTimedOut: restarted, but the alignment poll exhausted its budget. + restartVerifyTimedOut + // restartVerifyAligned: the only success -- restarted, live, aligned AND fresh. + restartVerifyAligned + // restartVerifyUnconfirmed: restarted and the poll returned, but the success gate was + // not met. No return of restartAndVerifyDaemon reaches it today (its five returns each + // land on one of the arms above); it is the fail-safe for a future producer return, and + // the arm every surface would fall into if a guard above it were weakened. + restartVerifyUnconfirmed + // restartVerifyOutcomeCount is the arity sentinel, NOT an outcome. It lets a test assert + // its table covers every constant, so a seventh outcome cannot be added and then + // silently rendered through three default arms. + restartVerifyOutcomeCount +) + +// classifyRestartVerify is the ONE classification of a restart+verify result. The branch +// ORDER is the contract, not an accident: the flags are not mutually exclusive, and +// testing TimedOut BEFORE the success conjunction is what stops a restart that ran out of +// budget from being reported as "now aligned". +// +// It takes a VALUE and has no nil arm. A nil *RestartVerifyResult means "no restart was +// attempted", which is not an outcome OF a restart, and the two pointer surfaces already +// answer it differently: the footer returns an empty line (upgradeFooterBody keys off that +// to print nothing at all), the log returns silently. +func classifyRestartVerify(rv RestartVerifyResult) restartVerifyOutcome { + switch { + case rv.Err != nil: + return restartVerifyError + case rv.LockPathUnknown: + return restartVerifyDeferredConfig + case rv.BackupWaitTimedOut: + return restartVerifyDeferredBackup + case rv.TimedOut: + return restartVerifyTimedOut + case rv.Restarted && rv.ProcessAlive && rv.Aligned && rv.FreshInfo: + return restartVerifyAligned + default: + return restartVerifyUnconfirmed + } +} + +// warn reports whether the outcome is a non-success, which is what the CLI upgrade footer +// styles on. Every non-success is a WARNING -- including a failed restart: an upgrade that +// installed the new binary but could not restart the daemon is not a failed upgrade, and +// neither front-end may style it as a hard error or let it change the exit code. +func (o restartVerifyOutcome) warn() bool { return o != restartVerifyAligned } + +// level is the same partition as warn, in the vocabulary the dashboard renders. Green only +// for the one success; every gap is yellow. It is single-sourced with warn so the two +// front-ends cannot disagree about the severity of one outcome, which they used to: the +// footer painted a failed restart yellow while the dashboard painted it red. Warn is the +// side that matches daemonStatusStyle, the repo's other daemon verdict, which has no red +// state at all. +func (o restartVerifyOutcome) level() orchestrator.HealthcheckSetupLevel { + if o == restartVerifyAligned { + return orchestrator.HealthcheckSetupLevelOk + } + return orchestrator.HealthcheckSetupLevelWarn +} diff --git a/cmd/proxsave/daemon_restart_verdict_test.go b/cmd/proxsave/daemon_restart_verdict_test.go new file mode 100644 index 00000000..d16e8ea8 --- /dev/null +++ b/cmd/proxsave/daemon_restart_verdict_test.go @@ -0,0 +1,174 @@ +package main + +import ( + "errors" + "strings" + "testing" + + "github.com/tis24dev/proxsave/internal/health" + "github.com/tis24dev/proxsave/internal/orchestrator" +) + +// restartVerdictFixtures is one representative result per outcome, in constant order. Each +// surface test walks it, so adding an outcome without a fixture fails the arity check rather +// than silently leaving a surface untested. +var restartVerdictFixtures = []struct { + outcome restartVerifyOutcome + rv RestartVerifyResult +}{ + {restartVerifyError, RestartVerifyResult{Err: errors.New("unit not found")}}, + {restartVerifyDeferredConfig, RestartVerifyResult{LockPathUnknown: true}}, + {restartVerifyDeferredBackup, RestartVerifyResult{BackupWaitTimedOut: true}}, + {restartVerifyTimedOut, RestartVerifyResult{Restarted: true, TimedOut: true}}, + {restartVerifyAligned, RestartVerifyResult{Restarted: true, ProcessAlive: true, Aligned: true, FreshInfo: true}}, + {restartVerifyUnconfirmed, RestartVerifyResult{Restarted: true}}, +} + +// TestLogUpgradeDaemonRestartWordsEveryOutcome is the FIRST test of the upgrade bootstrap log +// surface, which had none. It is half of the pair that classifies the same restart result on the +// same --upgrade run: the footer is read on the terminal, this one is what an unattended upgrade +// leaves behind on disk, so a divergence between them is only ever discovered afterwards. +// +// It also pins the severity split. The success is the ONLY line that goes out at INFO (Println, +// stdout); every gap is a WARNING (stderr). Routing them all through one emitter would be +// invisible on the terminal and would either lose the success line for anyone redirecting stderr +// away, or turn a normal upgrade into a false positive for anyone grepping stderr. +func TestLogUpgradeDaemonRestartWordsEveryOutcome(t *testing.T) { + want := map[restartVerifyOutcome]struct { + message string + warning bool + }{ + restartVerifyError: {"Daemon restart failed: unit not found (it may still run the old binary; restart it manually).", true}, + restartVerifyDeferredConfig: {"Config unreadable; daemon restart deferred. Restart when the config is readable or the daemon stays on the old binary.", true}, + restartVerifyDeferredBackup: {"A backup is running; daemon restart deferred. Restart when idle or the daemon stays on the old binary.", true}, + restartVerifyTimedOut: {"Daemon restarted but alignment check timeout", true}, + restartVerifyAligned: {"Daemon restarted and now aligned with the new binary.", false}, + restartVerifyUnconfirmed: {"Daemon restarted but alignment could not be confirmed", true}, + } + if len(want) != int(restartVerifyOutcomeCount) { + t.Fatalf("table covers %d outcomes, want all %d", len(want), restartVerifyOutcomeCount) + } + + for _, fixture := range restartVerdictFixtures { + expected, ok := want[fixture.outcome] + if !ok { + t.Fatalf("no expectation for outcome %d", fixture.outcome) + } + bootstrap, buf := captureBootstrapLog(t) + bootstrap.SetConsoleQuiet(true) + rv := fixture.rv + logUpgradeDaemonRestart(bootstrap, &rv) + got := buf.String() + if !strings.Contains(got, expected.message) { + t.Fatalf("outcome %d logged %q, want it to contain %q", fixture.outcome, got, expected.message) + } + if isWarning := strings.Contains(got, "WARNING"); isWarning != expected.warning { + t.Fatalf("outcome %d logged at warning=%v, want %v: %q", fixture.outcome, isWarning, expected.warning, got) + } + } + + // A nil result means no restart was attempted (an inactive daemon, or a cron host). It is + // not an outcome OF a restart, so the log must stay silent rather than invent one. + bootstrap, buf := captureBootstrapLog(t) + bootstrap.SetConsoleQuiet(true) + logUpgradeDaemonRestart(bootstrap, nil) + if buf.String() != "" { + t.Fatalf("a nil result must log nothing, got %q", buf.String()) + } +} + +// TestSummarizeRestartVerifyDistinguishesTheDeferrals pins the two deferral lines apart on the +// CLI footer. They are different remedies -- make the config readable, versus wait for the backup +// to finish -- but both lines merely contain "deferred", so the pre-existing substring assertion +// could not tell them apart and the config-unreadable arm had no coverage at all on any surface. +func TestSummarizeRestartVerifyDistinguishesTheDeferrals(t *testing.T) { + unknownLock := RestartVerifyResult{LockPathUnknown: true} + line, warn := summarizeRestartVerify(&unknownLock, "1.2.3") + if !warn { + t.Fatalf("a deferred restart is a warning, got warn=false") + } + if !strings.Contains(line, "config unreadable") || !strings.Contains(line, "restart when the config is readable") { + t.Fatalf("config-unreadable line wrong: %q", line) + } + if strings.Contains(line, "a backup is running") { + t.Fatalf("config-unreadable line must not name the backup deferral: %q", line) + } + + backup := RestartVerifyResult{BackupWaitTimedOut: true} + backupLine, _ := summarizeRestartVerify(&backup, "1.2.3") + if !strings.Contains(backupLine, "a backup is running") || strings.Contains(backupLine, "config unreadable") { + t.Fatalf("backup deferral line wrong: %q", backupLine) + } + if line == backupLine { + t.Fatalf("the two deferrals must not share a line: %q", line) + } +} + +// TestInstallVerifyVerdictNamesWhatItCouldNotEstablish covers the poll-only verdict, including +// the arm that used to lie. A daemon that IS process-alive but whose /proc alignment probe never +// returned a verdict was reported as "not running": wrong about the process, and silent about the +// fact that alignment -- not existence -- is what could not be established. An operator reading +// that after an install would go looking for a dead daemon that is running fine. +func TestInstallVerifyVerdictNamesWhatItCouldNotEstablish(t *testing.T) { + aliveUnverifiable := RestartVerifyResult{ProcessAlive: true} + level, keyword := installVerifyVerdict(aliveUnverifiable) + if level != orchestrator.HealthcheckSetupLevelWarn { + t.Fatalf("an unverifiable alignment is a warning, got level=%v", level) + } + if strings.Contains(keyword, "not running") { + t.Fatalf("a live daemon must not be reported as not running: %q", keyword) + } + if !strings.Contains(keyword, "running") || !strings.Contains(keyword, "could not be verified") { + t.Fatalf("the keyword must say it is running and that verification failed: %q", keyword) + } + + // The other three arms, so the split above cannot be widened over them. + aligned := RestartVerifyResult{ProcessAlive: true, Aligned: true, State: health.DaemonState{Version: "1.2.3", AlignChecked: true}} + if level, keyword := installVerifyVerdict(aligned); level != orchestrator.HealthcheckSetupLevelOk || + keyword != "running and aligned (v1.2.3)" { + t.Fatalf("aligned verdict wrong: level=%v keyword=%q", level, keyword) + } + behind := RestartVerifyResult{ProcessAlive: true, State: health.DaemonState{AlignChecked: true}} + if level, keyword := installVerifyVerdict(behind); level != orchestrator.HealthcheckSetupLevelWarn || + keyword != "running but not aligned (behind)" { + t.Fatalf("behind verdict wrong: level=%v keyword=%q", level, keyword) + } + // Genuinely down after the full poll budget: "not running" is a measured fact here, and the + // only arm entitled to say it. + down := RestartVerifyResult{TimedOut: true} + if level, keyword := installVerifyVerdict(down); level != orchestrator.HealthcheckSetupLevelWarn || + keyword != "not running" { + t.Fatalf("not-running verdict wrong: level=%v keyword=%q", level, keyword) + } +} + +// TestInstallVerifyVerdictIsNotTheRestartClassifier pins the separation the comment used to only +// assert. A poll-only result never carries Restarted, so routing it through classifyRestartVerify +// would put every install on a "restarted but..." arm -- the regression that once made the install +// always say "not confirmed" -- and would lose the BEHIND verdict, which that classifier has no +// arm for at all. +func TestInstallVerifyVerdictIsNotTheRestartClassifier(t *testing.T) { + // The shape verifyDaemonAligned returns for a healthy, aligned daemon: no Restarted flag. + pollOnlySuccess := RestartVerifyResult{ + ProcessAlive: true, + Aligned: true, + FreshInfo: true, // set from State.HaveInfo here, NOT from a new start timestamp + State: health.DaemonState{Version: "1.2.3", AlignChecked: true}, + } + if got := classifyRestartVerify(pollOnlySuccess); got == restartVerifyAligned { + t.Fatalf("the restart classifier must not call a poll-only result aligned, got %d", got) + } + if level, keyword := installVerifyVerdict(pollOnlySuccess); level != orchestrator.HealthcheckSetupLevelOk || + !strings.Contains(keyword, "running and aligned") { + t.Fatalf("the poll-only verdict must call it aligned: level=%v keyword=%q", level, keyword) + } + + // The BEHIND verdict has no counterpart in the restart classifier's six arms. + behind := RestartVerifyResult{ProcessAlive: true, State: health.DaemonState{AlignChecked: true}} + _, keyword := installVerifyVerdict(behind) + for _, rv := range restartVerdictFixtures { + if line, _ := summarizeRestartVerify(&rv.rv, ""); strings.Contains(line, keyword) { + t.Fatalf("restart outcome %d already words the behind verdict: %q", rv.outcome, line) + } + } +} diff --git a/cmd/proxsave/daemon_restart_verify.go b/cmd/proxsave/daemon_restart_verify.go index 414ec4e5..41871383 100644 --- a/cmd/proxsave/daemon_restart_verify.go +++ b/cmd/proxsave/daemon_restart_verify.go @@ -186,9 +186,11 @@ func restartAndVerifyDaemon(ctx context.Context, baseDir, lockFilePath string, l // verifyDaemonAligned is the poll-only variant (no restart, no backup-wait): it waits for an // ALREADY-(re)started daemon to become process-alive with an ASSESSABLE alignment, then returns // the state (res.Aligned tells aligned vs behind). It polls until ProcessAlive && AlignChecked -- -// NOT until Aligned -- so a daemon that is up but BEHIND is reported immediately (the SAME verdict -// --daemon-status gives), never as a timeout. TimedOut means the daemon never came up. There is no -// pre-restart snapshot, so FreshInfo simply reflects that an identity record exists. +// NOT until Aligned -- so a daemon that is up but BEHIND is reported immediately as behind, never +// as a timeout. TimedOut means the daemon never came up, or came up with an alignment that stayed +// unassessable. There is no pre-restart snapshot, so FreshInfo simply reflects that an identity +// record exists -- a DIFFERENT meaning from the restart path, where it means the process is new. +// installVerifyVerdict, not classifyRestartVerify, is what reads this result. func verifyDaemonAligned(ctx context.Context, baseDir string, interval time.Duration) RestartVerifyResult { if ctx == nil { ctx = context.Background() @@ -257,31 +259,36 @@ func daemonIsActive(ctx context.Context) bool { // summarizeRestartVerify renders a one-line, plain-text summary of a restart+verify // outcome for the upgrade footer (the CLI --upgrade path). version is the just-installed -// version, shown on the aligned line. Returns ("", false) when rv is nil (restart not -// attempted, e.g. the daemon was inactive). warn is true for any non-success outcome so -// the caller can style it as a warning WITHOUT ever changing the upgrade exit code. +// version, shown on the aligned line -- NOT rv.State.Version, which is what the running +// daemon reports and is what the dashboard shows instead. Returns ("", false) when rv is +// nil (restart not attempted, e.g. the daemon was inactive). warn is true for any +// non-success outcome so the caller can style it as a warning WITHOUT ever changing the +// upgrade exit code. The verdict itself comes from classifyRestartVerify, shared with the +// bootstrap log and the dashboard; only the wording below is this surface's own. func summarizeRestartVerify(rv *RestartVerifyResult, version string) (line string, warn bool) { if rv == nil { return "", false } - switch { - case rv.Err != nil: - return "Daemon: WARNING - restart failed: " + rv.Err.Error() + - " (the daemon may still run the old binary; restart it manually)", true - case rv.LockPathUnknown: - return "Daemon: WARNING - config unreadable; daemon restart deferred - " + - "restart when the config is readable or it stays on the old binary", true - case rv.BackupWaitTimedOut: - return "Daemon: WARNING - a backup is running; daemon restart deferred - " + - "restart when idle or it stays on the old binary", true - case rv.TimedOut: - return "Daemon: WARNING - restarted but alignment check timeout", true - case rv.Restarted && rv.ProcessAlive && rv.Aligned && rv.FreshInfo: + outcome := classifyRestartVerify(*rv) + switch outcome { + case restartVerifyError: + line = "Daemon: WARNING - restart failed: " + rv.Err.Error() + + " (the daemon may still run the old binary; restart it manually)" + case restartVerifyDeferredConfig: + line = "Daemon: WARNING - config unreadable; daemon restart deferred - " + + "restart when the config is readable or it stays on the old binary" + case restartVerifyDeferredBackup: + line = "Daemon: WARNING - a backup is running; daemon restart deferred - " + + "restart when idle or it stays on the old binary" + case restartVerifyTimedOut: + line = "Daemon: WARNING - restarted but alignment check timeout" + case restartVerifyAligned: + line = "Daemon: restarted, now aligned" if v := strings.TrimSpace(version); v != "" { - return "Daemon: restarted, now aligned (v" + v + ")", false + line = "Daemon: restarted, now aligned (v" + v + ")" } - return "Daemon: restarted, now aligned", false default: - return "Daemon: WARNING - restarted but alignment could not be confirmed", true + line = "Daemon: WARNING - restarted but alignment could not be confirmed" } + return line, outcome.warn() } diff --git a/cmd/proxsave/daemon_restart_verify_test.go b/cmd/proxsave/daemon_restart_verify_test.go index 0531a13a..7c299267 100644 --- a/cmd/proxsave/daemon_restart_verify_test.go +++ b/cmd/proxsave/daemon_restart_verify_test.go @@ -343,11 +343,108 @@ func TestRestartVerifyStatus(t *testing.T) { keyword != "RESTARTED, NOT CONFIRMED" { t.Fatalf("ambiguous status wrong: level=%v keyword=%q", level, keyword) } + // A failed restart is a WARNING, not an error: the new binary is already installed and the + // daemon merely still runs the old one, so it must not read as a hard failure. Yellow here + // matches the CLI upgrade footer, which has always painted this outcome yellow, and matches + // daemonStatusStyle, which has no red daemon state at all. failed := RestartVerifyResult{Err: errors.New("x")} - if level, keyword, msg := restartVerifyStatus(failed); level != orchestrator.HealthcheckSetupLevelError || + if level, keyword, msg := restartVerifyStatus(failed); level != orchestrator.HealthcheckSetupLevelWarn || keyword != "RESTART FAILED" || msg != "x" { t.Fatalf("failed status wrong: level=%v keyword=%q msg=%q", level, keyword, msg) } + // The config-unreadable deferral has its OWN keyword, distinct from the backup deferral -- + // they are different remedies (make the config readable vs wait for the backup). Both lines + // merely contain "deferred", so a substring assertion cannot tell them apart. + unknownLock := RestartVerifyResult{LockPathUnknown: true} + if level, keyword, _ := restartVerifyStatus(unknownLock); level != orchestrator.HealthcheckSetupLevelWarn || + keyword != "DEFERRED - CONFIG UNREADABLE" { + t.Fatalf("config-unreadable status wrong: level=%v keyword=%q", level, keyword) + } +} + +// TestRestartVerifySeverityMatchesFooter pins the rule that made the two front-ends agree: for +// EVERY outcome, the dashboard's level and the CLI footer's warn bool are the same statement. +// They used to disagree on exactly one outcome -- a failed restart was yellow on the CLI and red +// on the dashboard -- because each surface derived severity from its own switch. +func TestRestartVerifySeverityMatchesFooter(t *testing.T) { + results := []RestartVerifyResult{ + {Err: errors.New("boom")}, + {LockPathUnknown: true}, + {BackupWaitTimedOut: true}, + {Restarted: true, TimedOut: true}, + {Restarted: true, ProcessAlive: true, Aligned: true, FreshInfo: true}, + {Restarted: true}, + } + seen := map[restartVerifyOutcome]bool{} + for _, rv := range results { + outcome := classifyRestartVerify(rv) + seen[outcome] = true + _, warn := summarizeRestartVerify(&rv, "1.2.3") + level, _, _ := restartVerifyStatus(rv) + wantLevel := orchestrator.HealthcheckSetupLevelOk + if warn { + wantLevel = orchestrator.HealthcheckSetupLevelWarn + } + if level != wantLevel { + t.Fatalf("outcome %d: footer warn=%v but dashboard level=%v", outcome, warn, level) + } + // No daemon outcome is ever a hard error on either front-end. + if level == orchestrator.HealthcheckSetupLevelError { + t.Fatalf("outcome %d must not be Error", outcome) + } + } + if len(seen) != int(restartVerifyOutcomeCount) { + t.Fatalf("covered %d outcomes, want all %d", len(seen), restartVerifyOutcomeCount) + } +} + +// TestClassifyRestartVerifyPrecedence pins the branch ORDER, which is the contract the three +// surfaces now share and the one thing a merged classifier can silently get wrong. The flags are +// not mutually exclusive, so every fixture below sets several at once and asserts which one wins. +// Without this, reordering the shared switch moves all three surfaces consistently and quietly. +func TestClassifyRestartVerifyPrecedence(t *testing.T) { + full := RestartVerifyResult{ + Restarted: true, ProcessAlive: true, Aligned: true, FreshInfo: true, + TimedOut: true, BackupWaitTimedOut: true, LockPathUnknown: true, + } + withErr := full + withErr.Err = errors.New("boom") + if got := classifyRestartVerify(withErr); got != restartVerifyError { + t.Fatalf("Err must outrank every other flag, got %d", got) + } + if got := classifyRestartVerify(full); got != restartVerifyDeferredConfig { + t.Fatalf("LockPathUnknown must outrank the deferral and the success gate, got %d", got) + } + noLock := full + noLock.LockPathUnknown = false + if got := classifyRestartVerify(noLock); got != restartVerifyDeferredBackup { + t.Fatalf("BackupWaitTimedOut must outrank the success gate, got %d", got) + } + // The invariant the ordering exists to protect: a restart that ran out of budget is never + // reported as "now aligned", however complete the flags it collected look. + timedOutButLooksAligned := RestartVerifyResult{ + Restarted: true, ProcessAlive: true, Aligned: true, FreshInfo: true, TimedOut: true, + } + if got := classifyRestartVerify(timedOutButLooksAligned); got != restartVerifyTimedOut { + t.Fatalf("TimedOut must outrank the success gate, got %d", got) + } + // Each conjunct of the success gate is load-bearing: drop any one and it is not a success. + for _, drop := range []string{"Restarted", "ProcessAlive", "Aligned", "FreshInfo"} { + rv := RestartVerifyResult{Restarted: true, ProcessAlive: true, Aligned: true, FreshInfo: true} + switch drop { + case "Restarted": + rv.Restarted = false + case "ProcessAlive": + rv.ProcessAlive = false + case "Aligned": + rv.Aligned = false + case "FreshInfo": + rv.FreshInfo = false + } + if got := classifyRestartVerify(rv); got != restartVerifyUnconfirmed { + t.Fatalf("without %s the result must be unconfirmed, got %d", drop, got) + } + } } // TestBuildStatusPrompt: the shared styled result prompt carries the "Status: " label and the diff --git a/cmd/proxsave/daemon_setup.go b/cmd/proxsave/daemon_setup.go index e6c023f8..5057815e 100644 --- a/cmd/proxsave/daemon_setup.go +++ b/cmd/proxsave/daemon_setup.go @@ -186,9 +186,9 @@ func applyDaemonMode(ctx context.Context, cfg *config.Config, configPath, execTo } // verifyDaemonAlignedBestEffort waits (poll-only, no restart) for the just-(re)started daemon to -// become process-alive with an assessable alignment, then REPORTS its real state - the SAME verdict -// --daemon-status gives (aligned / behind / not running) - never a bare "timeout". It NEVER fails -// the caller (install / --daemon-setup): a behind or unconfirmed daemon is a warning, not an error. +// become process-alive with an assessable alignment, then REPORTS its real state - aligned, +// behind, alignment-unverifiable, or not running - never a bare "timeout". It NEVER fails the +// caller (install / --daemon-setup): a behind or unconfirmed daemon is a warning, not an error. func verifyDaemonAlignedBestEffort(ctx context.Context, baseDir string, interval time.Duration) RestartVerifyResult { logging.Info("Verifying daemon alignment...") rv := verifyDaemonAligned(ctx, baseDir, interval) @@ -200,12 +200,26 @@ func verifyDaemonAlignedBestEffort(ctx context.Context, baseDir string, interval return rv } -// installVerifyVerdict maps a poll-only verify result (verifyDaemonAligned) to the -// aligned / behind / not-running verdict as a (level, keyword) pair - the SAME verdict -// --daemon-status reports. Shared by the log line (verifyDaemonAlignedBestEffort) and the -// graphical install outcome (buildInstallOutcomePrompt) so they never diverge. It must NOT -// go through restartVerifyStatus, whose success arm needs Restarted/FreshInfo that the -// poll-only verify never sets - that mis-mapping made the install always say "not confirmed". +// installVerifyVerdict maps a poll-only verify result (verifyDaemonAligned) to a +// (level, keyword) pair, shared by the log line (verifyDaemonAlignedBestEffort) and the +// graphical install outcome (buildInstallOutcomePrompt) so the two never diverge. +// +// It must NOT go through classifyRestartVerify. That classifier's success arm requires +// Restarted, which the poll-only verify never sets (verifyDaemonAligned builds its result +// from a zero value and only ever assigns State/ProcessAlive/Aligned/FreshInfo/TimedOut), +// so every poll-only result would land on a "restarted but..." arm - the mis-mapping that +// once made the install always say "not confirmed". Four of that classifier's six arms are +// structurally unreachable here (no Err, no LockPathUnknown, no BackupWaitTimedOut, no +// Restarted), and its shape has no BEHIND verdict at all, which is the one verdict an +// install most needs to report. +// +// This is NOT the same verdict --daemon-status gives: daemonStatusStyle turns green only on +// a FRESH HEARTBEAT and ignores alignment, whereas a just-restarted daemon is aligned long +// before it writes its first heartbeat. The two answer different questions and word their +// "behind" differently on purpose. +// +// The keyword is interpolated mid-sentence by the log line ("Daemon %s."), which is why it +// is lower case unlike the dashboard's ALL-CAPS keywords. func installVerifyVerdict(rv RestartVerifyResult) (orchestrator.HealthcheckSetupLevel, string) { switch { case rv.ProcessAlive && rv.Aligned: @@ -216,6 +230,12 @@ func installVerifyVerdict(rv RestartVerifyResult) (orchestrator.HealthcheckSetup return orchestrator.HealthcheckSetupLevelOk, keyword case rv.ProcessAlive && rv.State.AlignChecked: return orchestrator.HealthcheckSetupLevelWarn, "running but not aligned (behind)" + case rv.ProcessAlive: + // Alive, but the /proc alignment probe never returned a verdict for it, so alignment + // is UNKNOWN (health.DaemonState gates every "behind" verdict on AlignChecked). + // Reporting "not running" here was wrong twice over: the process IS running, and the + // thing that could not be established was its ALIGNMENT, not its existence. + return orchestrator.HealthcheckSetupLevelWarn, "running, but alignment could not be verified" default: return orchestrator.HealthcheckSetupLevelWarn, "not running" } diff --git a/cmd/proxsave/dashboard.go b/cmd/proxsave/dashboard.go index f53ffbf9..c2864c8b 100644 --- a/cmd/proxsave/dashboard.go +++ b/cmd/proxsave/dashboard.go @@ -487,33 +487,42 @@ func runDashboardDaemonRestart(ctx context.Context, session *shell.Session, conf // restartVerifyStatus maps a restart+verify outcome to the styled daemon-result triple (a // shared HealthcheckSetupLevel + a short colored keyword + a one-line explanation), shared by -// the "Restart daemon" button and the post-upgrade restart. Success is green (Ok); a deferral -// (backup running), a not-confirmed alignment, and an ambiguous restart are yellow warnings -// (Warn); a restart error is red (Error). The explanation strings are unchanged from the old -// notice bodies -- only the outcome keyword is added for the colored "Status:" line. +// the "Restart daemon" button and the post-upgrade restart. The verdict and its severity come +// from classifyRestartVerify, shared with the CLI upgrade footer and the upgrade bootstrap +// log; only the keywords and explanations below are this surface's own. +// +// Success is green (Ok) and every gap -- including a failed restart -- is a yellow warning, +// matching both the CLI footer and daemonStatusStyle, the repo's other daemon verdict, which +// likewise has no red state. A restart that fails does not fail the upgrade: the new binary is +// already installed and the daemon simply still runs the old one. +// +// The version shown here is rv.State.Version, what the RUNNING daemon reports -- not the +// version the upgrade just installed, which is what summarizeRestartVerify shows instead. +// +// TimedOut and Unconfirmed deliberately render identically here (this screen has no useful +// distinction to draw between them), while the footer and the log word them differently. That +// is a rendering choice, which is why the two stay distinct outcomes upstream. func restartVerifyStatus(rv RestartVerifyResult) (orchestrator.HealthcheckSetupLevel, string, string) { - switch { - case rv.Err != nil: - return orchestrator.HealthcheckSetupLevelError, "RESTART FAILED", rv.Err.Error() - case rv.LockPathUnknown: - return orchestrator.HealthcheckSetupLevelWarn, "DEFERRED - CONFIG UNREADABLE", + outcome := classifyRestartVerify(rv) + switch outcome { + case restartVerifyError: + return outcome.level(), "RESTART FAILED", rv.Err.Error() + case restartVerifyDeferredConfig: + return outcome.level(), "DEFERRED - CONFIG UNREADABLE", "The config could not be read, so the real backup lock path is unknown; restart again once it is readable, or the daemon stays on the old binary." - case rv.BackupWaitTimedOut: - return orchestrator.HealthcheckSetupLevelWarn, "DEFERRED - BACKUP RUNNING", + case restartVerifyDeferredBackup: + return outcome.level(), "DEFERRED - BACKUP RUNNING", "Restart again once the backup finishes, or the daemon stays on the old binary." - case rv.TimedOut: - return orchestrator.HealthcheckSetupLevelWarn, "RESTARTED, NOT CONFIRMED", - "Open Daemon status to confirm it came back aligned." - case rv.Restarted && rv.ProcessAlive && rv.Aligned && rv.FreshInfo: + case restartVerifyAligned: // Success: the keyword ("RESTARTED, ALIGNED (vX)") already says everything, so no // explanation line -- a what-to-do suggestion only appears on a problem outcome. keyword := "RESTARTED, ALIGNED" if v := strings.TrimSpace(rv.State.Version); v != "" { keyword += " (v" + v + ")" } - return orchestrator.HealthcheckSetupLevelOk, keyword, "" - default: - return orchestrator.HealthcheckSetupLevelWarn, "RESTARTED, NOT CONFIRMED", + return outcome.level(), keyword, "" + default: // restartVerifyTimedOut and restartVerifyUnconfirmed + return outcome.level(), "RESTARTED, NOT CONFIRMED", "Open Daemon status to confirm it came back aligned." } } diff --git a/cmd/proxsave/install_finalize_stream_test.go b/cmd/proxsave/install_finalize_stream_test.go index 2a723135..9597f7cd 100644 --- a/cmd/proxsave/install_finalize_stream_test.go +++ b/cmd/proxsave/install_finalize_stream_test.go @@ -20,8 +20,8 @@ import ( ) // TestBuildInstallOutcomePromptVerified asserts the daemon-verified branch reuses the shared -// installVerifyVerdict/renderDaemonStatusLevel verdict (aligned / behind / not-running - the SAME -// as --daemon-status, NOT the restart-verify "not confirmed") and colors the permissions line by +// installVerifyVerdict/renderDaemonStatusLevel verdict (aligned / behind / unverifiable / +// not-running, NOT the restart-verify "not confirmed") and colors the permissions line by // status. ANSI is stripped so the assertions do not depend on the color profile. func TestBuildInstallOutcomePromptVerified(t *testing.T) { rv := RestartVerifyResult{ diff --git a/cmd/proxsave/install_outcome.go b/cmd/proxsave/install_outcome.go index c1758090..14d7738d 100644 --- a/cmd/proxsave/install_outcome.go +++ b/cmd/proxsave/install_outcome.go @@ -13,7 +13,7 @@ import ( // - installBanner (the same completed/aborted/failed title + severity the footer's // ANSI box uses), rendered here in the theme via renderInstallBanner; // - installVerifyVerdict + renderDaemonStatusLevel for the "Daemon:" line (the SAME -// aligned / behind / not-running verdict as --daemon-status); +// verdict the install's own log line reports, so the screen and the log agree); // - the same permStatus strings printInstallFooter switches on for "Permissions:". // // The graphical finalization is reached only on the success path (failures/aborts @@ -32,8 +32,8 @@ func buildInstallOutcomePrompt(rv RestartVerifyResult, verified bool, permStatus b.WriteString(theme.Text.Render("Daemon: ")) if verified { - // Same verdict the log line and --daemon-status show (NOT restartVerifyStatus, - // whose success arm needs Restarted/FreshInfo the poll-only verify never sets). + // Same verdict the install's log line shows (NOT classifyRestartVerify, whose + // success arm needs the Restarted flag the poll-only verify never sets). level, keyword := installVerifyVerdict(rv) b.WriteString(renderDaemonStatusLevel(level, keyword)) } else { diff --git a/cmd/proxsave/upgrade.go b/cmd/proxsave/upgrade.go index 37c9cfa9..68a394e2 100644 --- a/cmd/proxsave/upgrade.go +++ b/cmd/proxsave/upgrade.go @@ -92,20 +92,26 @@ func upgradeBackupLockPath(configPath, baseDir string) (string, bool) { } // logUpgradeDaemonRestart mirrors the restart outcome to the bootstrap log (the CLI -// path). It never fails the upgrade -- every outcome is informational. +// path). It never fails the upgrade -- every outcome is informational. It classifies the +// SAME value the upgrade footer summarizes, on the same run, through the same shared +// classifyRestartVerify, so the log and the footer cannot report different verdicts. +// +// Only the success line goes through Println (stdout, INFO); every gap goes through +// Warning (stderr). That split is this surface's own and is not part of the verdict. func logUpgradeDaemonRestart(bootstrap *logging.BootstrapLogger, rv *RestartVerifyResult) { - switch { - case rv == nil: + if rv == nil { return - case rv.Err != nil: + } + switch classifyRestartVerify(*rv) { + case restartVerifyError: bootstrap.Warning("Daemon restart failed: %v (it may still run the old binary; restart it manually).", rv.Err) - case rv.LockPathUnknown: + case restartVerifyDeferredConfig: bootstrap.Warning("Config unreadable; daemon restart deferred. Restart when the config is readable or the daemon stays on the old binary.") - case rv.BackupWaitTimedOut: + case restartVerifyDeferredBackup: bootstrap.Warning("A backup is running; daemon restart deferred. Restart when idle or the daemon stays on the old binary.") - case rv.TimedOut: + case restartVerifyTimedOut: bootstrap.Warning("Daemon restarted but alignment check timeout") - case rv.Restarted && rv.ProcessAlive && rv.Aligned && rv.FreshInfo: + case restartVerifyAligned: bootstrap.Println("Daemon restarted and now aligned with the new binary.") default: bootstrap.Warning("Daemon restarted but alignment could not be confirmed") From 98247d333dfa6bc5a76977b97c28a64aba34ea39 Mon Sep 17 00:00:00 2001 From: tis24dev Date: Mon, 3 Aug 2026 15:45:36 +0200 Subject: [PATCH 16/50] refactor(daemon): one spelling of the systemd unit name across both front-ends daemonUnitName exists, and every systemctl call already used it -- but seven production sites wrote the string out by hand, and the split ran along the front-end boundary. The CLI printed "Daemon service (%s)" from the constant while the dashboard hardcoded the same line; the CLI's "Daemon mode enabled: %s is active..." existed twice with the constant and once without. That is not a tidiness problem. Renaming or versioning the unit would move the systemctl calls, which all take the constant, while leaving the on-screen text naming a unit that no longer exists -- and it would do so on ONE front-end, so the operator would be told two different things depending on which one they opened. The unit path had the same defect one level down: it spelled the name a second time, so a rename would have written the new unit to the old file name and then enabled a unit the writer never created. The rendered output does not change. Every replacement is a constant expression, so the compiler folds it back to the same literal: all six user-visible messages are present with identical counts in a binary built before and after. Only the unit-name token moves. The dashboard and the CLI still word the same event differently ("The resident daemon (X) is active. The cron entry was removed." versus "Daemon mode enabled: X is active and the cron entry was removed."), and that stays exactly as it is -- wording is presentation, and this change is about the fact underneath it. Left alone deliberately: the two --daemon help strings in internal/cli, which cannot reach a constant in package main. The systemd unit helpers live in cmd by design (internal/health is documented as never shelling out to systemctl), so moving the name into an internal package to serve two lines of static help prose would push a deployment detail across a boundary the repo drew on purpose. Tests: an AST walk asserting the name appears as a string literal exactly once in the whole package -- at its own declaration. It reads real literals, not comments or doc strings, and excludes test files, where asserting the rendered line SHOULD name what the operator sees. Verified by mutation: 8 one-line edits, all 8 killed by a named test. --- cmd/proxsave/daemon_service.go | 9 ++- cmd/proxsave/daemon_setup.go | 4 +- cmd/proxsave/daemon_unit_name_test.go | 104 ++++++++++++++++++++++++++ cmd/proxsave/dashboard.go | 8 +- 4 files changed, 118 insertions(+), 7 deletions(-) create mode 100644 cmd/proxsave/daemon_unit_name_test.go diff --git a/cmd/proxsave/daemon_service.go b/cmd/proxsave/daemon_service.go index 58166a03..dc3f3ee3 100644 --- a/cmd/proxsave/daemon_service.go +++ b/cmd/proxsave/daemon_service.go @@ -16,7 +16,14 @@ import ( ) const ( + // daemonUnitName is the SINGLE spelling of the systemd unit. Every systemctl call, + // every on-screen line naming the service, and the unit path below derive from it, + // on both front-ends -- the dashboard used to hardcode the string in four places + // while the CLI took it from here, so renaming the unit would have left the two + // disagreeing about what to tell the operator. The only literal is this line. daemonUnitName = "proxsave-daemon.service" + // daemonUnitDir is where systemd reads host-local unit files from. + daemonUnitDir = "/etc/systemd/system" // daemonExecPath is the canonical entrypoint symlink the unit invokes (same // path used for the crontab line); resolved by ensureGoSymlink at install. daemonExecPath = "/usr/local/bin/proxsave" @@ -24,7 +31,7 @@ const ( // daemonUnitPath is the systemd unit path. A var (not const) so tests can point it // at a temp dir. -var daemonUnitPath = "/etc/systemd/system/proxsave-daemon.service" +var daemonUnitPath = filepath.Join(daemonUnitDir, daemonUnitName) // buildDaemonUnit renders the systemd unit. systemd is only the keep-alive // supervisor (Restart=always); the daemon schedules internally. A non-empty diff --git a/cmd/proxsave/daemon_setup.go b/cmd/proxsave/daemon_setup.go index 5057815e..6cb54a82 100644 --- a/cmd/proxsave/daemon_setup.go +++ b/cmd/proxsave/daemon_setup.go @@ -333,12 +333,12 @@ func maybeAutoMigrateDaemon(ctx context.Context, configPath, baseDir, execToken bootstrap.Println("Daemon mode was previously removed (--daemon-remove); leaving the cron scheduler in place.") return } - bootstrap.Println("Migrating to the resident daemon scheduler (proxsave-daemon.service)...") + bootstrap.Println("Migrating to the resident daemon scheduler (" + daemonUnitName + ")...") if err := applyDaemonMode(ctx, cfg, configPath, execToken, bootstrap); err != nil { bootstrap.Warning("Daemon migration failed; staying on cron: %v", err) return } - bootstrap.Println("Daemon mode enabled: proxsave-daemon.service is active and the cron entry was removed.") + bootstrap.Println("Daemon mode enabled: " + daemonUnitName + " is active and the cron entry was removed.") } // setBackupEnvKeys reads backup.env, applies the given key=value edits (replacing diff --git a/cmd/proxsave/daemon_unit_name_test.go b/cmd/proxsave/daemon_unit_name_test.go new file mode 100644 index 00000000..3de07b0f --- /dev/null +++ b/cmd/proxsave/daemon_unit_name_test.go @@ -0,0 +1,104 @@ +package main + +import ( + "go/ast" + "go/parser" + "go/token" + "os" + "path/filepath" + "strconv" + "strings" + "testing" +) + +// TestDaemonUnitNameHasOneSpelling is the structural pin for D6. The systemd unit name is a +// deployment fact the command owns once, in daemonUnitName -- but seven production sites wrote +// it out by hand, and the split ran along the front-end boundary: the CLI took "Daemon service +// (%s)" from the constant while the dashboard hardcoded the same line, and the CLI's +// "Daemon mode enabled: %s is active..." existed twice with the constant and once without. +// +// The damage from that is not cosmetic. Renaming or versioning the unit would move the +// systemctl calls (which all use the constant) while leaving the on-screen text naming a unit +// that no longer exists, and it would do so on ONE front-end -- the operator would be told two +// different things depending on which one they opened. +// +// A grep-based check would pass on a comment or a doc string, so this walks the AST and looks +// only at real string literals. Test files are excluded on purpose: a test asserting the +// rendered line SHOULD name the string the operator sees, which is the whole point of pinning +// it here. +func TestDaemonUnitNameHasOneSpelling(t *testing.T) { + entries, err := os.ReadDir(".") + if err != nil { + t.Fatalf("read package dir: %v", err) + } + + fset := token.NewFileSet() + var declPos token.Pos + var offenders []string + + for _, entry := range entries { + name := entry.Name() + if entry.IsDir() || !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") { + continue + } + file, err := parser.ParseFile(fset, name, nil, 0) + if err != nil { + t.Fatalf("parse %s: %v", name, err) + } + + // Remember where the constant declares the name, so it is not its own offender. + for _, decl := range file.Decls { + gen, ok := decl.(*ast.GenDecl) + if !ok || gen.Tok != token.CONST { + continue + } + for _, spec := range gen.Specs { + value, ok := spec.(*ast.ValueSpec) + if !ok { + continue + } + for i, ident := range value.Names { + if ident.Name == "daemonUnitName" && i < len(value.Values) { + declPos = value.Values[i].Pos() + } + } + } + } + + ast.Inspect(file, func(node ast.Node) bool { + lit, ok := node.(*ast.BasicLit) + if !ok || lit.Kind != token.STRING { + return true + } + text, err := strconv.Unquote(lit.Value) + if err != nil || !strings.Contains(text, "proxsave-daemon.service") { + return true + } + if lit.Pos() == declPos { + return true + } + offenders = append(offenders, fset.Position(lit.Pos()).String()) + return true + }) + } + + if declPos == token.NoPos { + t.Fatal("daemonUnitName is not declared as a constant any more; this test can no longer tell the declaration from a stray literal") + } + if len(offenders) > 0 { + t.Fatalf("the unit name must come from daemonUnitName, not a literal; found %d:\n %s", + len(offenders), strings.Join(offenders, "\n ")) + } +} + +// TestDaemonUnitPathDerivesFromTheName pins the unit PATH to the same source. The path used to +// spell the unit name a second time, so a rename would have written the new unit to the old +// file name -- systemctl would then enable a unit the writer never created. +func TestDaemonUnitPathDerivesFromTheName(t *testing.T) { + if got, want := filepath.Base(daemonUnitPath), daemonUnitName; got != want { + t.Fatalf("daemonUnitPath names %q, want the unit %q", got, want) + } + if dir := filepath.Dir(daemonUnitPath); dir != daemonUnitDir { + t.Fatalf("daemonUnitPath lives in %q, want %q", dir, daemonUnitDir) + } +} diff --git a/cmd/proxsave/dashboard.go b/cmd/proxsave/dashboard.go index c2864c8b..4715840b 100644 --- a/cmd/proxsave/dashboard.go +++ b/cmd/proxsave/dashboard.go @@ -477,7 +477,7 @@ func runDashboardDaemonRestart(ctx context.Context, session *shell.Session, conf lockPath, lockKnown = backupLockFilePath(cfg, baseDir) } var rv RestartVerifyResult - _ = components.RunTask(ctx, session, "Restarting daemon", "Restarting proxsave-daemon.service...", func(taskCtx context.Context, report func(string)) error { + _ = components.RunTask(ctx, session, "Restarting daemon", "Restarting "+daemonUnitName+"...", func(taskCtx context.Context, report func(string)) error { rv = restartAndVerifyDaemon(taskCtx, baseDir, lockPath, lockKnown, interval) return nil }) @@ -592,10 +592,10 @@ func runDashboardDaemonAdmin(ctx context.Context, session *shell.Session, instal doneMsg := "Reverted to the cron scheduler and removed the daemon service. Future upgrades will not reinstall it." if install { title = "Installing daemon" - work = "Installing and enabling proxsave-daemon.service..." + work = "Installing and enabling " + daemonUnitName + "..." doneTitle = "Daemon installed" doneKeyword = "INSTALLED" - doneMsg = "The resident daemon (proxsave-daemon.service) is active. The cron entry was removed." + doneMsg = "The resident daemon (" + daemonUnitName + ") is active. The cron entry was removed." } execToken := daemonSelfExecPath() var opErr error @@ -761,7 +761,7 @@ func buildDaemonStatusPrompt(level orchestrator.HealthcheckSetupLevel, keyword, b.WriteString("\n") b.WriteString(theme.Text.Render("Scheduler mode: " + components.SanitizeText(mode))) b.WriteString("\n") - b.WriteString(theme.Text.Render("Daemon service (proxsave-daemon.service): " + unit)) + b.WriteString(theme.Text.Render("Daemon service (" + daemonUnitName + "): " + unit)) b.WriteString("\n") b.WriteString(theme.Text.Render("Service state (systemctl is-active): " + components.SanitizeText(active))) b.WriteString("\n") From 4550677bab2570e325c583d44d98dc6b061e9604 Mon Sep 17 00:00:00 2001 From: tis24dev Date: Mon, 3 Aug 2026 16:07:06 +0200 Subject: [PATCH 17/50] fix(install): one abort policy for optional steps, so a TUI Ctrl+C stops the install The two install drivers answered "did the user ask to stop?" with different tests, and each knew only its own. The CLI tested ctx.Err(), which is non-nil after a SIGINT raised by Ctrl+C at a cooked-mode stdin prompt. The TUI tested shell.ErrClosed, which is what the Charm session resolves to when the program ends -- and in the alternate screen the terminal is in raw mode, so Ctrl+C arrives as a KEY, not as a signal. The repo's own tests show the two are distinct errors (TestAskReturnsErrClosedOnCtrlC versus the context.Canceled case in ask_test.go), so neither test subsumes the other. Three of the TUI's optional steps had no test at all. RunPostInstallAudit, RunTelegramSetup and RunHealthcheckSetup captured their error, logged it as "failed (non-blocking)", and let the driver walk on. The end guard checks only ctx.Err(), so it did not catch a session death either. The run then reached RunStreamTask, which cancels the task context as soon as Ask fails, so the finalization executed on a dead context and installed no scheduler -- and the deferred footer still printed "Installation completed". That false green is exactly what the CLI's own rule was written to prevent; its comment says so in as many words. Same reasoning, applied on one side only. Only the self-params step was guarded, and only against session death, because it took a local fatal = mapUIDeath. In self mode the next step therefore caught the abort by accident while every other mode did not -- an inconsistency that is itself evidence the gap was an oversight rather than a decision. optionalInstallStepAborts is now the single rule, holding both discriminators, and every optional step on both drivers asks it. Benign failures still continue: an EOF at a prompt, an unreachable relay, a failed dry-run. Those steps are accessory and the config is already written. Each surface keeps its own wording -- the CLI still says "interrupted by signal", which is accurate there, and the TUI says "interrupted", which covers a key. Only the decision is shared. Tests: the full discriminator matrix, the canonical-error contract the footer keys off, and a structural pin that walks runInstallTUI, collects the error of every flow call it makes, and requires each to reach an abort decision. A test of the rule could not have caught the original defect, because the rule was never wrong -- three steps simply had no rule. Only a test of the wiring can, and a newly added step that forgets the guard now fails by name. Verified by mutation: 10 one-line edits, all 10 killed by a named test. --- cmd/proxsave/install.go | 9 +- cmd/proxsave/install_step_abort.go | 59 ++++++++ cmd/proxsave/install_step_abort_test.go | 170 ++++++++++++++++++++++++ cmd/proxsave/install_tui.go | 28 ++-- cmd/proxsave/install_tui_selfhc_test.go | 24 ++-- 5 files changed, 267 insertions(+), 23 deletions(-) create mode 100644 cmd/proxsave/install_step_abort.go create mode 100644 cmd/proxsave/install_step_abort_test.go diff --git a/cmd/proxsave/install.go b/cmd/proxsave/install.go index a0b13365..f24b1131 100644 --- a/cmd/proxsave/install.go +++ b/cmd/proxsave/install.go @@ -204,11 +204,12 @@ func runInstall(ctx context.Context, configPath string, bootstrap *logging.Boots // (false green). So propagate an aborted-install error instead, which renders // the "Installation aborted" banner exactly like the TUI (mapUIDeath). // -// ctx.Err() is the authoritative discriminator: the run context is WithCancel and -// is cancelled only by the signal handler (setupRunContext), so it is non-nil -// after Ctrl+C but nil for a plain EOF at an optional prompt. +// The discrimination itself lives in optionalInstallStepAborts, shared with the TUI +// driver so the two front-ends cannot answer the same question differently. On this +// path the deciding signal is always ctx.Err(): Ctrl+C at a cooked-mode stdin prompt +// raises a real signal, and no Charm session exists to produce a shell.ErrClosed. func skipOptionalInstallStepOnAbort(ctx context.Context, bootstrap *logging.BootstrapLogger, step string, err error) error { - if ctx.Err() != nil { + if optionalInstallStepAborts(ctx, err) { if bootstrap != nil { bootstrap.Warning("%s aborted (interrupted by signal); stopping the install before finalization", step) } diff --git a/cmd/proxsave/install_step_abort.go b/cmd/proxsave/install_step_abort.go new file mode 100644 index 00000000..8cb1effc --- /dev/null +++ b/cmd/proxsave/install_step_abort.go @@ -0,0 +1,59 @@ +package main + +import ( + "context" + "errors" + + "github.com/tis24dev/proxsave/internal/logging" + "github.com/tis24dev/proxsave/internal/ui/shell" +) + +// optionalInstallStepAborts is the SINGLE rule deciding whether a failed OPTIONAL install +// step (post-install audit, Telegram pairing, healthcheck setup, healthcheck self params) +// stops the install or is merely skipped. Both drivers ask it; neither re-derives it. +// +// The rule needs TWO discriminators, and each driver used to know only its own: +// +// - ctx.Err() != nil -- the run context is WithCancel and is cancelled only by the +// signal handler (setupRunContext), so it is non-nil after a SIGINT/SIGTERM. This is +// what the stdin CLI sees, where Ctrl+C at a cooked-mode prompt raises a real signal. +// - shell.ErrClosed -- the Charm program terminated. This is what the TUI sees: in the +// alternate screen the terminal is in raw mode, so Ctrl+C arrives as a KEY that ends +// the program rather than as a signal, and Ask resolves to ErrClosed (pinned by +// TestAskReturnsErrClosedOnCtrlC). It is a DIFFERENT error from context.Canceled. +// +// Knowing only one of the two is what made the TUI continue past a user's Ctrl+C: three +// of its optional steps logged "failed (non-blocking)" and carried on to a finalization +// that then ran on a context RunStreamTask had already cancelled, installing no scheduler +// -- and the run still ended on the "Installation completed" banner. That false green is +// precisely what the CLI's own rule was written to prevent. +// +// Anything else -- a plain EOF at a prompt, an unreachable relay, a failed dry-run -- is +// benign: the step is accessory, the config is already written, so the install continues. +// A nil error is never an abort. +func optionalInstallStepAborts(ctx context.Context, err error) bool { + if err == nil { + return false + } + if ctx != nil && ctx.Err() != nil { + return true + } + return errors.Is(err, shell.ErrClosed) +} + +// abortInstallOnOptionalStep is the TUI driver's action on that verdict: return the +// canonical aborted-install error so the deferred footer shows the honest banner, or nil +// to carry on. It does NOT close the session -- the driver's deferred Close already runs +// before the deferred footer (defers are LIFO), so the terminal is released in time. +// +// The benign case is deliberately silent here: each step already logs its own outcome +// (logTelegramSetupOutcome, logHealthcheckSetupOutcome, the audit block), and duplicating +// that would put two lines in the install log for one event. +func abortInstallOnOptionalStep(ctx context.Context, bootstrap *logging.BootstrapLogger, step string, err error) error { + if !optionalInstallStepAborts(ctx, err) { + return nil + } + // Worded for the surface: on the TUI the cause is usually a key, not a signal. + logBootstrapWarning(bootstrap, "%s aborted (interrupted); stopping the install before finalization", step) + return wrapInstallError(errInteractiveAborted) +} diff --git a/cmd/proxsave/install_step_abort_test.go b/cmd/proxsave/install_step_abort_test.go new file mode 100644 index 00000000..241f2a67 --- /dev/null +++ b/cmd/proxsave/install_step_abort_test.go @@ -0,0 +1,170 @@ +package main + +import ( + "context" + "errors" + "fmt" + "go/ast" + "go/parser" + "go/token" + "io" + "strings" + "testing" + + "github.com/tis24dev/proxsave/internal/installer" + "github.com/tis24dev/proxsave/internal/ui/shell" +) + +// TestOptionalInstallStepAbortsKnowsBothDiscriminators is the matrix both drivers now +// share. Each front-end used to know only one column of it: the CLI tested ctx.Err() and +// was blind to a Charm session death, the TUI tested shell.ErrClosed and was blind to a +// cancelled run context. The two are genuinely different errors -- the shell resolves a +// raw-mode Ctrl+C to ErrClosed, not to context.Canceled -- so neither test subsumes the +// other and a step guarded by only one of them has a hole. +func TestOptionalInstallStepAbortsKnowsBothDiscriminators(t *testing.T) { + cancelled, cancel := context.WithCancel(context.Background()) + cancel() + live := context.Background() + + cases := []struct { + name string + ctx context.Context + err error + want bool + }{ + {"no error is never an abort", live, nil, false}, + {"no error is not an abort even on a dead context", cancelled, nil, false}, + {"a benign EOF at a prompt continues", live, io.EOF, false}, + {"a user cancel continues", live, installer.ErrInstallCancelled, false}, + {"an unreachable relay continues", live, errors.New("dial tcp: connection refused"), false}, + {"a signal aborts", cancelled, io.EOF, true}, + {"a session death aborts", live, shell.ErrClosed, true}, + {"a wrapped session death aborts", live, fmt.Errorf("telegram: %w", shell.ErrClosed), true}, + {"a nil context is tolerated", nil, io.EOF, false}, + {"a nil context still sees a session death", nil, shell.ErrClosed, true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := optionalInstallStepAborts(tc.ctx, tc.err); got != tc.want { + t.Fatalf("optionalInstallStepAborts = %v, want %v", got, tc.want) + } + }) + } +} + +// TestAbortInstallOnOptionalStepReturnsTheCanonicalError checks the TUI-side action: the +// abort must be the SAME error the mandatory steps raise, because that is what the +// deferred footer keys off to print "Installation aborted" instead of "completed". +func TestAbortInstallOnOptionalStepReturnsTheCanonicalError(t *testing.T) { + bootstrap, buf := captureBootstrapLog(t) + bootstrap.SetConsoleQuiet(true) + + if err := abortInstallOnOptionalStep(context.Background(), bootstrap, "Telegram setup", io.EOF); err != nil { + t.Fatalf("a benign error must not abort, got %v", err) + } + if buf.String() != "" { + t.Fatalf("the benign case must stay silent (the step logs its own outcome), got %q", buf.String()) + } + + err := abortInstallOnOptionalStep(context.Background(), bootstrap, "Telegram setup", shell.ErrClosed) + if err == nil || !errors.Is(err, errInteractiveAborted) { + t.Fatalf("want errInteractiveAborted, got %v", err) + } + if !isInstallAbortedError(err) { + t.Fatalf("the footer's own predicate must recognise it: %v", err) + } + if got := buf.String(); !strings.Contains(got, "Telegram setup aborted") || + !strings.Contains(got, "stopping the install before finalization") { + t.Fatalf("abort not recorded in the install log: %q", got) + } +} + +// TestEveryTUIInstallStepIsGuarded is the structural pin. The original defect was not a +// wrong rule, it was three steps that had no rule at all: RunPostInstallAudit, +// RunTelegramSetup and RunHealthcheckSetup captured an error, logged it as +// "non-blocking", and let the driver walk on to a finalization that then ran on a dead +// context and installed no scheduler -- while the run still ended on the "Installation +// completed" banner. +// +// A test of the rule cannot catch that; only a test of the WIRING can. This walks +// runInstallTUI, collects the error identifier of every flow call it makes, and requires +// each one to reach an abort decision. A newly added step that forgets the guard fails +// here by name. +func TestEveryTUIInstallStepIsGuarded(t *testing.T) { + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, "install_tui.go", nil, 0) + if err != nil { + t.Fatalf("parse install_tui.go: %v", err) + } + + var driver *ast.FuncDecl + for _, decl := range file.Decls { + if fn, ok := decl.(*ast.FuncDecl); ok && fn.Name.Name == "runInstallTUI" { + driver = fn + } + } + if driver == nil { + t.Fatal("runInstallTUI not found; this test can no longer see the driver it pins") + } + + // Error identifiers produced by a flow call, and identifiers that reach a decision. + stepErrors := map[string]token.Pos{} + guarded := map[string]bool{} + + ast.Inspect(driver, func(node ast.Node) bool { + switch n := node.(type) { + case *ast.AssignStmt: + // , := flowinstall.Something(...) + if len(n.Rhs) != 1 || len(n.Lhs) != 2 { + return true + } + call, ok := n.Rhs[0].(*ast.CallExpr) + if !ok { + return true + } + sel, ok := call.Fun.(*ast.SelectorExpr) + if !ok { + return true + } + pkg, ok := sel.X.(*ast.Ident) + if !ok || pkg.Name != "flowinstall" { + return true + } + if ident, ok := n.Lhs[1].(*ast.Ident); ok && ident.Name != "_" { + stepErrors[ident.Name] = n.Pos() + } + case *ast.CallExpr: + // abortInstallOnOptionalStep(ctx, bootstrap, "step", ) or mapUIDeath() + name := "" + switch fn := n.Fun.(type) { + case *ast.Ident: + name = fn.Name + case *ast.SelectorExpr: + name = fn.Sel.Name + } + if name != "abortInstallOnOptionalStep" && name != "mapUIDeath" { + return true + } + for _, arg := range n.Args { + if ident, ok := arg.(*ast.Ident); ok { + guarded[ident.Name] = true + } + } + } + return true + }) + + if len(stepErrors) == 0 { + t.Fatal("no flowinstall step calls found in runInstallTUI; the matcher has gone stale") + } + var unguarded []string + for name, pos := range stepErrors { + if !guarded[name] { + unguarded = append(unguarded, fmt.Sprintf("%s (%s)", name, fset.Position(pos))) + } + } + if len(unguarded) > 0 { + t.Fatalf("every install step's error must reach an abort decision; %d do not:\n %s", + len(unguarded), strings.Join(unguarded, "\n ")) + } +} diff --git a/cmd/proxsave/install_tui.go b/cmd/proxsave/install_tui.go index 10cb810d..4f24d558 100644 --- a/cmd/proxsave/install_tui.go +++ b/cmd/proxsave/install_tui.go @@ -21,14 +21,17 @@ import ( var runHealthcheckSelfParamsFn = flowinstall.RunHealthcheckSelfParams // applySelfHealthcheckParams runs the optional self-mode healthcheck-params step. -// It returns a non-nil error ONLY when the step must abort the whole install -// (session death, via fatal = mapUIDeath); a user cancel or any other step error -// is non-blocking - the step is skipped with a warning, matching the sibling -// optional install steps. -func applySelfHealthcheckParams(ctx context.Context, session *shell.Session, baseDir, configPath string, bootstrap *logging.BootstrapLogger, fatal func(error) error) error { +// It returns a non-nil error ONLY when the step must abort the whole install; a user +// cancel or any other step error is non-blocking - the step is skipped with a warning, +// matching the sibling optional install steps. +// +// The abort decision is optionalInstallStepAborts, shared with the CLI driver. It used +// to be a locally passed fatal = mapUIDeath, which knew only about session death and so +// missed a cancelled run context. +func applySelfHealthcheckParams(ctx context.Context, session *shell.Session, baseDir, configPath string, bootstrap *logging.BootstrapLogger) error { if err := runHealthcheckSelfParamsFn(ctx, session, baseDir, configPath); err != nil { - if mapped := fatal(err); errors.Is(mapped, errInteractiveAborted) { - return mapped + if abortErr := abortInstallOnOptionalStep(ctx, bootstrap, "Healthcheck self params", err); abortErr != nil { + return abortErr } if bootstrap != nil { bootstrap.Warning("Healthcheck self params failed (non-blocking): %v", err) @@ -233,6 +236,9 @@ func runInstallTUI(ctx context.Context, configPath string, bootstrap *logging.Bo // based on actionable warning hints like "set BACKUP_*=false to disable". if !skipConfigWizard { auditRes, auditErr := flowinstall.RunPostInstallAudit(ctx, session, execInfo.ExecPath, configPath, false) + if abortErr := abortInstallOnOptionalStep(ctx, bootstrap, "Post-install audit", auditErr); abortErr != nil { + return abortErr + } if bootstrap != nil { if auditErr != nil { bootstrap.Warning("Post-install check failed (non-blocking): %v", auditErr) @@ -270,6 +276,9 @@ func runInstallTUI(ctx context.Context, configPath string, bootstrap *logging.Bo // returns Shown=false without any UI when Telegram is not centrally enabled. if !skipConfigWizard { telegramRes, telegramErr := flowinstall.RunTelegramSetup(ctx, session, baseDir, configPath, false) + if abortErr := abortInstallOnOptionalStep(ctx, bootstrap, "Telegram setup", telegramErr); abortErr != nil { + return abortErr + } logTelegramSetupOutcome(bootstrap, telegramRes, telegramErr) // Self-mode healthchecks: collect the ping URLs BEFORE the healthcheck @@ -277,7 +286,7 @@ func runInstallTUI(ctx context.Context, configPath string, bootstrap *logging.Bo // written HEALTHCHECK_ALIVE_URL). Only when self was chosen in the wizard. if wizardData != nil && wizardData.HealthcheckMode == "self" { logging.DebugStepBootstrap(bootstrap, "install workflow (tui)", "healthcheck self params") - if err := applySelfHealthcheckParams(ctx, session, baseDir, configPath, bootstrap, mapUIDeath); err != nil { + if err := applySelfHealthcheckParams(ctx, session, baseDir, configPath, bootstrap); err != nil { return err } } @@ -287,6 +296,9 @@ func runInstallTUI(ctx context.Context, configPath string, bootstrap *logging.Bo // (self) verify the pasted alive URL is reachable. Eligibility is decided solely // by RunHealthcheckSetup (re-reads the written config); Shown=false with no UI otherwise. hcRes, hcErr := flowinstall.RunHealthcheckSetup(ctx, session, baseDir, configPath, false) + if abortErr := abortInstallOnOptionalStep(ctx, bootstrap, "Healthcheck setup", hcErr); abortErr != nil { + return abortErr + } logHealthcheckSetupOutcome(bootstrap, hcRes, hcErr) } diff --git a/cmd/proxsave/install_tui_selfhc_test.go b/cmd/proxsave/install_tui_selfhc_test.go index 2ed44d39..44ece364 100644 --- a/cmd/proxsave/install_tui_selfhc_test.go +++ b/cmd/proxsave/install_tui_selfhc_test.go @@ -9,33 +9,35 @@ import ( "github.com/tis24dev/proxsave/internal/ui/shell" ) +// TestApplySelfHealthcheckParamsCancelContinues covers the optional self-params step's +// abort decision. The cancelled-context row is the one the old local rule missed: it +// knew only about session death, so a SIGTERM during this step was demoted to a warning +// and the install carried on to finalization. func TestApplySelfHealthcheckParamsCancelContinues(t *testing.T) { orig := runHealthcheckSelfParamsFn t.Cleanup(func() { runHealthcheckSelfParamsFn = orig }) - // fatal mimics mapUIDeath: only a session close is fatal. - fatal := func(err error) error { - if errors.Is(err, shell.ErrClosed) { - return wrapInstallError(errInteractiveAborted) - } - return err - } + cancelled, cancel := context.WithCancel(context.Background()) + cancel() cases := []struct { name string + ctx context.Context stepErr error wantAbort bool }{ - {"cancel is non-blocking", installer.ErrInstallCancelled, false}, - {"session death aborts", shell.ErrClosed, true}, - {"success", nil, false}, + {"cancel is non-blocking", context.Background(), installer.ErrInstallCancelled, false}, + {"session death aborts", context.Background(), shell.ErrClosed, true}, + {"a cancelled run context aborts", cancelled, installer.ErrInstallCancelled, true}, + {"success", context.Background(), nil, false}, + {"success is not an abort even on a dead context", cancelled, nil, false}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { runHealthcheckSelfParamsFn = func(context.Context, *shell.Session, string, string) error { return tc.stepErr } - err := applySelfHealthcheckParams(context.Background(), nil, "/base", "/cfg", nil, fatal) + err := applySelfHealthcheckParams(tc.ctx, nil, "/base", "/cfg", nil) if tc.wantAbort { if err == nil || !errors.Is(err, errInteractiveAborted) { t.Fatalf("want abort (errInteractiveAborted), got %v", err) From f58de69b2b76d0b7f587a203520078084cdd9fe3 Mon Sep 17 00:00:00 2001 From: tis24dev Date: Mon, 3 Aug 2026 16:25:10 +0200 Subject: [PATCH 18/50] docs: publish exit code 17, and pin the exit-code contract against drift ExitGuardsPending (17) shipped with the --cleanup-guards verdict and was in neither reference. Worse, both went on asserting that 16 was the only non-zero code that does not mean a failure -- CLI_REFERENCE in a note, TROUBLESHOOTING in a sentence counting "three of them". Exit codes are a scripting interface: the docs are the only place an operator can learn what a code means, so a wrapper written from them would have paged someone for a datastore that was merely still mounted, and would have treated it the same as a cleanup that genuinely failed. Those are different remedies -- unmount and retry, versus report a bug. Both documents now carry the row and say which codes are benign. The prose is fixed alongside the table because the sentence is the part people actually read. The pin is two tests, and they are deliberately not a list of codes: a test that repeats the list drifts exactly the way the docs did. The first reads the constants out of internal/types/exit_codes.go with the AST and requires a table row for each, in both files; the second requires the benign-code prose to name every one of them and refuses the pre-17 claims by their exact wording. Verified by mutation: 6 one-line edits, all 6 killed. The one that matters is adding a hypothetical code 18 to the types package with no documentation -- that fails the test with no help from anyone, which is the case this exists for. --- cmd/proxsave/exit_codes_doc_drift_test.go | 130 ++++++++++++++++++++++ docs/CLI_REFERENCE.md | 7 +- docs/TROUBLESHOOTING.md | 13 ++- 3 files changed, 143 insertions(+), 7 deletions(-) create mode 100644 cmd/proxsave/exit_codes_doc_drift_test.go diff --git a/cmd/proxsave/exit_codes_doc_drift_test.go b/cmd/proxsave/exit_codes_doc_drift_test.go new file mode 100644 index 00000000..1725da13 --- /dev/null +++ b/cmd/proxsave/exit_codes_doc_drift_test.go @@ -0,0 +1,130 @@ +package main + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "os" + "regexp" + "strconv" + "strings" + "testing" +) + +// exitCodeDocs are the two references that publish the exit-code contract. Both carry a +// table AND a prose sentence counting how many non-zero codes are not failures, so both +// go stale together when a code is added. +var exitCodeDocs = []struct { + name string + path string +}{ + {"CLI_REFERENCE.md", "../../docs/CLI_REFERENCE.md"}, + {"TROUBLESHOOTING.md", "../../docs/TROUBLESHOOTING.md"}, +} + +// TestExitCodesAreDocumented pins the exit-code contract against doc drift. Exit codes +// are a SCRIPTING interface: an operator gates a cron wrapper or a monitoring probe on +// them, and the only place they can learn what a code means is these two files. A code +// that exists in the binary but not in the docs is indistinguishable, from outside, from +// a code that means "something broke". +// +// This is not hypothetical. ExitGuardsPending (17) shipped with --cleanup-guards and was +// absent from both documents, which went on asserting that 16 was the only non-zero code +// that does not mean a failure -- so a wrapper written from the docs would have paged +// someone for a datastore that was merely still mounted. +// +// The constants are read from the source rather than listed here on purpose: a test that +// repeats the list drifts in exactly the same way the docs did. +func TestExitCodesAreDocumented(t *testing.T) { + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, "../../internal/types/exit_codes.go", nil, 0) + if err != nil { + t.Fatalf("parse exit_codes.go: %v", err) + } + + codes := map[string]int{} + for _, decl := range file.Decls { + gen, ok := decl.(*ast.GenDecl) + if !ok || gen.Tok != token.CONST { + continue + } + for _, spec := range gen.Specs { + value, ok := spec.(*ast.ValueSpec) + if !ok || len(value.Names) != 1 || len(value.Values) != 1 { + continue + } + lit, ok := value.Values[0].(*ast.BasicLit) + if !ok || lit.Kind != token.INT { + continue + } + n, convErr := strconv.Atoi(lit.Value) + if convErr != nil { + continue + } + codes[value.Names[0].Name] = n + } + } + if len(codes) < 10 { + t.Fatalf("found only %d exit-code constants; the matcher has gone stale", len(codes)) + } + // The interrupted code lives in main (128 + SIGINT), not in the types package, but it + // is part of the same published contract. + codes["exitCodeInterrupted"] = exitCodeInterrupted + + for _, doc := range exitCodeDocs { + body, readErr := os.ReadFile(doc.path) + if readErr != nil { + t.Fatalf("read %s: %v", doc.name, readErr) + } + text := string(body) + var missing []string + for name, code := range codes { + // A table row for the code: "| `17` | ... |". Matching the row rather than the + // bare number avoids passing on an unrelated "17" elsewhere in the prose. + row := regexp.MustCompile(`(?m)^\|\s*` + "`" + strconv.Itoa(code) + "`" + `\s*\|`) + if !row.MatchString(text) { + missing = append(missing, fmt.Sprintf("%s (%d)", name, code)) + } + } + if len(missing) > 0 { + t.Errorf("%s has no table row for %d exit code(s): %s", + doc.name, len(missing), strings.Join(missing, ", ")) + } + } +} + +// TestExitCodeProseCountsTheBenignCodes pins the sentence a reader actually acts on. +// Both documents state, in prose, how many non-zero codes are NOT failures. That count +// is what tells someone whether their "|| alert" wrapper is safe, and it is what went +// wrong last time: the table could have been fixed while the sentence still said the +// wrong number, and the sentence is the part people read. +// +// The benign non-zero codes today are 1 (a run that succeeded with warnings), 16 +// (nothing to back up), 17 (guards still holding the storage) and 130 (cancelled). +func TestExitCodeProseCountsTheBenignCodes(t *testing.T) { + benign := []string{"`1`", "`16`", "`17`", "`130`"} + + for _, doc := range exitCodeDocs { + body, err := os.ReadFile(doc.path) + if err != nil { + t.Fatalf("read %s: %v", doc.name, err) + } + text := string(body) + + // The claim that one specific code is the only benign one must not survive. + for _, stale := range []string{ + "`16` is the one non-zero code", + "Three of them are not failures", + } { + if strings.Contains(text, stale) { + t.Errorf("%s still carries the pre-17 claim %q", doc.name, stale) + } + } + for _, code := range benign { + if !strings.Contains(text, code) { + t.Errorf("%s never mentions benign exit code %s", doc.name, code) + } + } + } +} diff --git a/docs/CLI_REFERENCE.md b/docs/CLI_REFERENCE.md index ed1ecd13..dcf00d2f 100644 --- a/docs/CLI_REFERENCE.md +++ b/docs/CLI_REFERENCE.md @@ -804,11 +804,14 @@ USE_COLOR=false proxsave | `14` | security error | Errors detected by the security check | | `15` | encryption error | Error during encryption setup or processing | | `16` | backup skipped | No backup was performed, for a benign reason: another backup already held the lock, or `BACKUP_ENABLED=false`. Not a failure | +| `17` | guards still in place | `--cleanup-guards` only. The cleanup itself ran fine, but the storage is still locked: guard mounts or immutable flags are left behind (typically hidden under a live mount), or the remaining count could not be confirmed. Also returned by `--cleanup-guards --dry-run` when it finds guards. Not a failure — unmount the datastore and retry | | `130` | interrupted | The run was cancelled with Ctrl+C (128 plus SIGINT) | -**Note**: `16` is the one non-zero code that does not mean something went wrong. A +**Note**: `16` and `17` are the non-zero codes that do not mean something went wrong. A wrapper of the form `proxsave --backup || alert` will page you every time two runs -overlap unless it excludes it. +overlap unless it excludes `16`. For `--cleanup-guards`, `17` is the one to act on but +not to report as a bug: `1` there means the cleanup itself failed, which is a different +remedy. **Note**: Cloud storage is non-critical. A cloud upload failure does **not** abort the run with a storage error (`5`): the local backup is kept, but the failure is recorded as a diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index 5d9b437e..30f90d8e 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -1125,11 +1125,13 @@ proxsave --support ## Exit Codes `proxsave` returns a specific exit code so scripts and the daemon can react to the -failure class. Three of them are not failures: `1` is also what a backup that succeeded -with warnings returns, `16` means no backup was performed for a benign reason, and -`130` means the run was cancelled by hand. Only `2` through `15` are unambiguous -failures. A wrapper that treats "non-zero" as "page someone" raises false alarms on -warning-only runs, on overlapping runs, on paused hosts, and on anything you Ctrl+C. +failure class. Four of them are not failures: `1` is also what a backup that succeeded +with warnings returns, `16` means no backup was performed for a benign reason, `17` +means a guard cleanup ran fine but the storage is still locked, and `130` means the run +was cancelled by hand. Only `2` through `15` are unambiguous failures. A wrapper that +treats "non-zero" as "page someone" raises false alarms on warning-only runs, on +overlapping runs, on paused hosts, on a datastore that is simply still mounted, and on +anything you Ctrl+C. | Code | Name | Meaning | |------|------|---------| @@ -1150,6 +1152,7 @@ warning-only runs, on overlapping runs, on paused hosts, and on anything you Ctr | `14` | security error | The security check reported errors | | `15` | encryption error | Error during encryption setup or processing | | `16` | backup skipped | No backup was performed, for a benign reason: another backup already held the lock, or `BACKUP_ENABLED=false`. Not a failure | +| `17` | guards still in place | `--cleanup-guards` only. The cleanup ran without error but guard mounts or immutable flags are left behind, or the remaining count could not be confirmed. Also returned by `--cleanup-guards --dry-run` when it finds guards. Not a failure: unmount the datastore and retry. A `1` from that mode means the cleanup itself failed | | `130` | interrupted | The run was cancelled with Ctrl+C (128 plus SIGINT) | A backup that finishes with warnings (no errors) is promoted from `0` to exit `1` From 5619e024a731ce9b696e8ea020451a5fac2a3b82 Mon Sep 17 00:00:00 2001 From: tis24dev Date: Mon, 3 Aug 2026 16:25:10 +0200 Subject: [PATCH 19/50] refactor(orchestrator): delete the error-only guard-cleanup wrapper cleanupMountGuards had two exported wrappers. --cleanup-guards took the error-only one, so a run that removed nothing because the datastore was still mounted returned a nil error and the mode exited 0, telling a gating script the storage was unlocked while guards were still holding it. The report that would have said otherwise was computed on every call and thrown away at the door. The verdict work moved the mode onto the report, which fixed the symptom and left the wrapper behind with no production callers -- still exported, still the shorter of the two names, still the one a hurried caller would reach for. It is gone. The report is now the engine's only way in, so discarding that state is a visible choice at each call site rather than the default. Its twenty callers were all tests in this package, which can reach the private cleanupMountGuards directly; nothing outside orchestrator referenced it. A structural test keeps it deleted. Re-adding a convenience wrapper is an easy, well-meant change, and no behavioural test fails when an unused one appears -- only a test of the shape can catch that. It asserts exactly one exported function calls cleanupMountGuards and that it returns the report, not just an error. Verified by mutation: both the wrapper coming back and the entry point losing its report are killed by it. Also updates the six comments that named the deleted function, including the two that recount this history and would otherwise point at something no longer in the tree. --- cmd/proxsave/cleanup_guards_verdict.go | 7 +- cmd/proxsave/main_modes.go | 7 +- internal/orchestrator/guards_cleanup.go | 19 ++--- .../guards_cleanup_dryrun_honesty_test.go | 8 +-- .../guards_cleanup_entrypoint_test.go | 71 +++++++++++++++++++ internal/orchestrator/guards_cleanup_test.go | 16 ++--- internal/orchestrator/mount_guard_apply.go | 2 +- .../orchestrator/mount_guard_chattr_index.go | 6 +- .../mount_guard_chattr_index_test.go | 56 +++++++-------- 9 files changed, 133 insertions(+), 59 deletions(-) create mode 100644 internal/orchestrator/guards_cleanup_entrypoint_test.go diff --git a/cmd/proxsave/cleanup_guards_verdict.go b/cmd/proxsave/cleanup_guards_verdict.go index 2209b13c..52681242 100644 --- a/cmd/proxsave/cleanup_guards_verdict.go +++ b/cmd/proxsave/cleanup_guards_verdict.go @@ -13,9 +13,10 @@ import ( // // Both drive orchestrator.CleanupMountGuardsReport, but they used to read it // differently: the dashboard classified the report (CLEAN/FOUND, DONE/PENDING) while -// --cleanup-guards took the error-only CleanupMountGuards wrapper and exited 0 for -// anything short of an engine failure. A script gating on the exit code was told the -// storage was unlocked while guards were still holding it. +// --cleanup-guards took an error-only wrapper beside it and exited 0 for anything short +// of an engine failure. A script gating on the exit code was told the storage was +// unlocked while guards were still holding it. That wrapper has since been removed, so +// the report is the engine's only exported entry point. // // What is shared here is the CLASSIFICATION and the FACTS. The call to action is not: // the dashboard names a button ("Apply"), the CLI names a flag, and neither wording diff --git a/cmd/proxsave/main_modes.go b/cmd/proxsave/main_modes.go index 03647035..1fbfa812 100644 --- a/cmd/proxsave/main_modes.go +++ b/cmd/proxsave/main_modes.go @@ -204,9 +204,10 @@ func runCleanupGuardsMode(ctx context.Context, args *cli.Args, bootstrap *loggin } logger := logging.New(level, false) - // The REPORT, not the error-only CleanupMountGuards wrapper: exiting 0 with guards - // still holding the storage actively misleads a script gating on the exit code, and - // the read that would have told us was being thrown away. Same seam the dashboard + // The REPORT: exiting 0 with guards still holding the storage actively misleads a + // script gating on the exit code, and the read that would have told us was being + // thrown away. This mode used to take an error-only wrapper that did exactly that; + // the wrapper is gone, so the report is now the only way in. Same seam the dashboard // uses, so one stub covers both front-ends in tests. report, err := cleanupGuardsReport(ctx, logger, args.DryRun) if err != nil { diff --git a/internal/orchestrator/guards_cleanup.go b/internal/orchestrator/guards_cleanup.go index 183460df..b519656e 100644 --- a/internal/orchestrator/guards_cleanup.go +++ b/internal/orchestrator/guards_cleanup.go @@ -95,19 +95,20 @@ func (r GuardCleanupReport) HasGuards() bool { return r.BindGuards > 0 || r.ImmutableGuards > 0 } -// CleanupMountGuards removes ProxSave mount guards created under mountGuardBaseDir. +// CleanupMountGuardsReport removes ProxSave mount guards created under mountGuardBaseDir +// and reports what it found and did. In dry-run it is a read-only CHECK (it reports what +// is present without changing anything); a real run reports what it removed and what is +// left pending. // // Safety: this will only unmount guard bind mounts when they are the currently-visible // mount on the mountpoint (i.e. the guard is the top-most mount at that mountpoint). // If a real mount is stacked on top, the guard will be left in place. -func CleanupMountGuards(ctx context.Context, logger *logging.Logger, dryRun bool) error { - _, err := cleanupMountGuards(ctx, logger, dryRun) - return err -} - -// CleanupMountGuardsReport is CleanupMountGuards with a structured report of what was -// found/done. In dry-run it is a read-only CHECK (reports what is present without -// changing anything); a real run reports what it removed and what is left pending. +// +// It is the ONLY exported entry point, deliberately. An error-only twin used to sit +// beside it, and --cleanup-guards took that one: a run that removed nothing because the +// datastore was still mounted returned a nil error and the mode exited 0, telling a +// gating script the storage was unlocked when it was not. Handing every caller the +// report is what makes throwing that state away a visible choice rather than the default. func CleanupMountGuardsReport(ctx context.Context, logger *logging.Logger, dryRun bool) (GuardCleanupReport, error) { return cleanupMountGuards(ctx, logger, dryRun) } diff --git a/internal/orchestrator/guards_cleanup_dryrun_honesty_test.go b/internal/orchestrator/guards_cleanup_dryrun_honesty_test.go index b3cc0b41..a4ce84b6 100644 --- a/internal/orchestrator/guards_cleanup_dryrun_honesty_test.go +++ b/internal/orchestrator/guards_cleanup_dryrun_honesty_test.go @@ -37,8 +37,8 @@ func TestCleanupMountGuards_MissingLeafNotCountedWouldClear(t *testing.T) { logger := logging.New(types.LogLevelInfo, false) var buf bytes.Buffer logger.SetOutput(&buf) - if err := CleanupMountGuards(context.Background(), logger, true); err != nil { - t.Fatalf("CleanupMountGuards dry-run: %v", err) + if _, err := cleanupMountGuards(context.Background(), logger, true); err != nil { + t.Fatalf("cleanupMountGuards dry-run: %v", err) } if len(*ran) != 0 { t.Fatalf("dry-run must run no chattr, calls=%#v", *ran) @@ -71,8 +71,8 @@ func TestCleanupMountGuards_DryRunOutOfAllowlistPending(t *testing.T) { logger := logging.New(types.LogLevelInfo, false) var buf bytes.Buffer logger.SetOutput(&buf) - if err := CleanupMountGuards(context.Background(), logger, true); err != nil { - t.Fatalf("CleanupMountGuards dry-run: %v", err) + if _, err := cleanupMountGuards(context.Background(), logger, true); err != nil { + t.Fatalf("cleanupMountGuards dry-run: %v", err) } if len(*ran) != 0 { t.Fatalf("dry-run must run no chattr, calls=%#v", *ran) diff --git a/internal/orchestrator/guards_cleanup_entrypoint_test.go b/internal/orchestrator/guards_cleanup_entrypoint_test.go new file mode 100644 index 00000000..fe64a045 --- /dev/null +++ b/internal/orchestrator/guards_cleanup_entrypoint_test.go @@ -0,0 +1,71 @@ +package orchestrator + +import ( + "go/ast" + "go/parser" + "go/token" + "strings" + "testing" +) + +// TestGuardCleanupHasOneExportedEntryPoint pins the shape that fixed the --cleanup-guards +// exit-code defect, rather than the defect's symptom. +// +// cleanupMountGuards used to have two exported wrappers: one returning the report, one +// returning only an error. --cleanup-guards took the error-only one, so a run that removed +// nothing because the datastore was still mounted returned nil and the mode exited 0 -- +// telling a gating script the storage was unlocked while guards were still holding it. The +// state that would have said otherwise was computed and then discarded at the call site. +// +// Deleting that wrapper is what makes the mistake unavailable, and only a structural test +// keeps it deleted: re-adding a convenience wrapper is an easy, well-meant change, and no +// behavioural test fails when an unused one appears. This asserts the engine offers exactly +// one exported way in and that it hands the caller the report. +func TestGuardCleanupHasOneExportedEntryPoint(t *testing.T) { + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, "guards_cleanup.go", nil, 0) + if err != nil { + t.Fatalf("parse guards_cleanup.go: %v", err) + } + + var exported []string + for _, decl := range file.Decls { + fn, ok := decl.(*ast.FuncDecl) + if !ok || fn.Recv != nil || !fn.Name.IsExported() || fn.Body == nil { + continue + } + calls := false + ast.Inspect(fn.Body, func(node ast.Node) bool { + call, ok := node.(*ast.CallExpr) + if !ok { + return true + } + if ident, ok := call.Fun.(*ast.Ident); ok && ident.Name == "cleanupMountGuards" { + calls = true + } + return true + }) + if !calls { + continue + } + exported = append(exported, fn.Name.Name) + + // The one entry point must return the report, not just an error. + if fn.Type.Results == nil || len(fn.Type.Results.List) != 2 { + t.Fatalf("%s must return (GuardCleanupReport, error); a report-less entry point is what let "+ + "--cleanup-guards exit 0 with guards still in place", fn.Name.Name) + } + first, ok := fn.Type.Results.List[0].Type.(*ast.Ident) + if !ok || first.Name != "GuardCleanupReport" { + t.Fatalf("%s returns %v first, want GuardCleanupReport", fn.Name.Name, fn.Type.Results.List[0].Type) + } + } + + if len(exported) != 1 { + t.Fatalf("guard cleanup must have exactly ONE exported entry point, found %d: %s", + len(exported), strings.Join(exported, ", ")) + } + if exported[0] != "CleanupMountGuardsReport" { + t.Fatalf("the entry point is %s, want CleanupMountGuardsReport", exported[0]) + } +} diff --git a/internal/orchestrator/guards_cleanup_test.go b/internal/orchestrator/guards_cleanup_test.go index 8b0718b6..f8b182ac 100644 --- a/internal/orchestrator/guards_cleanup_test.go +++ b/internal/orchestrator/guards_cleanup_test.go @@ -89,8 +89,8 @@ func TestCleanupMountGuards_UnmountsVisibleAndRemovesDirWhenNoRemaining(t *testi } logger := logging.New(types.LogLevelError, false) - if err := CleanupMountGuards(context.Background(), logger, false); err != nil { - t.Fatalf("CleanupMountGuards error: %v", err) + if _, err := cleanupMountGuards(context.Background(), logger, false); err != nil { + t.Fatalf("cleanupMountGuards error: %v", err) } if len(unmounted) != 1 || unmounted[0] != "/mnt/visible" { t.Fatalf("unmounted=%#v want [\"/mnt/visible\"]", unmounted) @@ -134,8 +134,8 @@ func TestCleanupMountGuards_DoesNotUnmountHiddenGuards(t *testing.T) { } logger := logging.New(types.LogLevelError, false) - if err := CleanupMountGuards(context.Background(), logger, false); err != nil { - t.Fatalf("CleanupMountGuards error: %v", err) + if _, err := cleanupMountGuards(context.Background(), logger, false); err != nil { + t.Fatalf("cleanupMountGuards error: %v", err) } } @@ -185,8 +185,8 @@ func TestCleanupMountGuards_RereadFailureKeepsDir(t *testing.T) { } logger := logging.New(types.LogLevelError, false) - if err := CleanupMountGuards(context.Background(), logger, false); err != nil { - t.Fatalf("CleanupMountGuards must be non-fatal on reread failure, got %v", err) + if _, err := cleanupMountGuards(context.Background(), logger, false); err != nil { + t.Fatalf("cleanupMountGuards must be non-fatal on reread failure, got %v", err) } if removed { t.Fatalf("guard directory must be kept when the verification reread fails (fail-closed)") @@ -230,8 +230,8 @@ func TestCleanupMountGuards_RereadFailureSummaryUnknown(t *testing.T) { logger := logging.New(types.LogLevelInfo, false) var buf bytes.Buffer logger.SetOutput(&buf) - if err := CleanupMountGuards(context.Background(), logger, false); err != nil { - t.Fatalf("CleanupMountGuards: %v", err) + if _, err := cleanupMountGuards(context.Background(), logger, false); err != nil { + t.Fatalf("cleanupMountGuards: %v", err) } out := buf.String() if !strings.Contains(out, "guards-remaining=unknown") { diff --git a/internal/orchestrator/mount_guard_apply.go b/internal/orchestrator/mount_guard_apply.go index fa8487b5..fbe0984a 100644 --- a/internal/orchestrator/mount_guard_apply.go +++ b/internal/orchestrator/mount_guard_apply.go @@ -262,7 +262,7 @@ func (a *pbsMountGuardApply) protectOfflineTarget(guardTarget string) { // recreation on this mountpoint is still skipped by the storage-mount preflight // (shouldSkipUnmountedStorageMount), and the config-only restore never extracts // into datastore mountpoints, so this only means EXTERNAL writers are not blocked -// while the storage is offline. CleanupMountGuards still clears any legacy +// while the storage is offline. CleanupMountGuardsReport still clears any legacy // chattr +i flags recorded by older ProxSave versions. func (a *pbsMountGuardApply) warnOfflineTargetUnguarded(guardTarget string, bindErr error) { a.warning("PBS mount guard: could NOT guard offline mountpoint %s (read-only bind mount failed: %v). "+ diff --git a/internal/orchestrator/mount_guard_chattr_index.go b/internal/orchestrator/mount_guard_chattr_index.go index 09f2d850..54f71e3f 100644 --- a/internal/orchestrator/mount_guard_chattr_index.go +++ b/internal/orchestrator/mount_guard_chattr_index.go @@ -11,7 +11,7 @@ import ( // mountGuardChattrTargetsName is the index file (under mountGuardBaseDir) that // records mountpoint directories ProxSave marked immutable via the `chattr +i` -// fallback guard during restore. CleanupMountGuards reads it back to clear +// fallback guard during restore. CleanupMountGuardsReport reads it back to clear // exactly those directories (and only those) with `chattr -i`. // // Unlike the read-only bind-mount guard — which a later real mount simply shadows @@ -82,7 +82,7 @@ func parseImmutableGuardTargets(data []byte) []string { } // recordImmutableGuardTarget appends target to the immutable-guard index after a -// successful `chattr +i`, so CleanupMountGuards can later clear exactly the +// successful `chattr +i`, so CleanupMountGuardsReport can later clear exactly the // directories ProxSave itself made immutable. Best-effort: any failure is logged // and swallowed so a recording problem never aborts the restore. The write is // atomic and de-duplicated. @@ -90,7 +90,7 @@ func parseImmutableGuardTargets(data []byte) []string { // Retained on purpose, not currently wired into a production code path. The // offline mount-guard fallback is warn-only now (it no longer applies `chattr +i`), // so nothing in restore writes the index anymore. The reader side stays fully live: -// CleanupMountGuards / warnLegacyImmutableGuards still parse and clear entries left +// CleanupMountGuardsReport / warnLegacyImmutableGuards still parse and clear entries left // by older ProxSave versions. This writer is kept (and exercised by tests) as the // canonical definition of the index format the reader consumes, so the `chattr +i` // fallback can be re-enabled without re-deriving it. diff --git a/internal/orchestrator/mount_guard_chattr_index_test.go b/internal/orchestrator/mount_guard_chattr_index_test.go index ec4ebb54..48a12e04 100644 --- a/internal/orchestrator/mount_guard_chattr_index_test.go +++ b/internal/orchestrator/mount_guard_chattr_index_test.go @@ -316,8 +316,8 @@ func TestCleanupMountGuards_ClearsImmutableWhenNotMounted(t *testing.T) { withTempGuardBaseDir(t) ran := installChattrCleanupSeams(t, []byte("/mnt/pve/offline\n/media/usb\n"), "", nil) - if err := CleanupMountGuards(context.Background(), newTestLogger(), false); err != nil { - t.Fatalf("CleanupMountGuards: %v", err) + if _, err := cleanupMountGuards(context.Background(), newTestLogger(), false); err != nil { + t.Fatalf("cleanupMountGuards: %v", err) } want := map[string]bool{"chattr -i /mnt/pve/offline": true, "chattr -i /media/usb": true} if len(*ran) != 2 || !want[(*ran)[0]] || !want[(*ran)[1]] { @@ -332,8 +332,8 @@ func TestCleanupMountGuards_SkipsImmutableWhenMounted(t *testing.T) { mountinfo := "36 35 0:30 / /mnt/pve/offline rw - nfs server:/export rw\n" ran := installChattrCleanupSeams(t, []byte("/mnt/pve/offline\n"), mountinfo, nil) - if err := CleanupMountGuards(context.Background(), newTestLogger(), false); err != nil { - t.Fatalf("CleanupMountGuards: %v", err) + if _, err := cleanupMountGuards(context.Background(), newTestLogger(), false); err != nil { + t.Fatalf("cleanupMountGuards: %v", err) } if len(*ran) != 0 { t.Fatalf("chattr -i must not run for a mounted (shadowed) target, calls=%#v", *ran) @@ -345,8 +345,8 @@ func TestCleanupMountGuards_DryRunNoChattr(t *testing.T) { withTempGuardBaseDir(t) ran := installChattrCleanupSeams(t, []byte("/mnt/pve/offline\n"), "", nil) - if err := CleanupMountGuards(context.Background(), newTestLogger(), true); err != nil { - t.Fatalf("CleanupMountGuards dry-run: %v", err) + if _, err := cleanupMountGuards(context.Background(), newTestLogger(), true); err != nil { + t.Fatalf("cleanupMountGuards dry-run: %v", err) } if len(*ran) != 0 { t.Fatalf("dry-run must not run chattr -i, calls=%#v", *ran) @@ -359,8 +359,8 @@ func TestCleanupMountGuards_RejectsNonDatastoreIndexEntry(t *testing.T) { withTempGuardBaseDir(t) ran := installChattrCleanupSeams(t, []byte("/etc/passwd-dir\n/var/lib/x\n/mnt/pve/legit\n"), "", nil) - if err := CleanupMountGuards(context.Background(), newTestLogger(), false); err != nil { - t.Fatalf("CleanupMountGuards: %v", err) + if _, err := cleanupMountGuards(context.Background(), newTestLogger(), false); err != nil { + t.Fatalf("cleanupMountGuards: %v", err) } if len(*ran) != 1 || (*ran)[0] != "chattr -i /mnt/pve/legit" { t.Fatalf("only the datastore-root entry may be cleared, calls=%#v", *ran) @@ -374,7 +374,7 @@ func TestCleanupMountGuards_ChattrFailureNonFatalContinues(t *testing.T) { ran := installChattrCleanupSeams(t, []byte("/mnt/pve/first\n/mnt/pve/second\n"), "", map[string]error{"chattr -i /mnt/pve/first": errors.New("operation not permitted")}) - if err := CleanupMountGuards(context.Background(), newTestLogger(), false); err != nil { + if _, err := cleanupMountGuards(context.Background(), newTestLogger(), false); err != nil { t.Fatalf("per-target chattr failure must be non-fatal, got %v", err) } if len(*ran) != 2 || (*ran)[0] != "chattr -i /mnt/pve/first" || (*ran)[1] != "chattr -i /mnt/pve/second" { @@ -387,8 +387,8 @@ func TestCleanupMountGuards_MissingIndexNoOp(t *testing.T) { withTempGuardBaseDir(t) ran := installChattrCleanupSeams(t, nil, "", nil) - if err := CleanupMountGuards(context.Background(), newTestLogger(), false); err != nil { - t.Fatalf("CleanupMountGuards: %v", err) + if _, err := cleanupMountGuards(context.Background(), newTestLogger(), false); err != nil { + t.Fatalf("cleanupMountGuards: %v", err) } if len(*ran) != 0 { t.Fatalf("missing index must be a no-op, calls=%#v", *ran) @@ -403,8 +403,8 @@ func TestCleanupMountGuards_IsMountedErrorSkips(t *testing.T) { // Force isMounted to error: both /proc reads fail. mountGuardReadFile = func(string) ([]byte, error) { return nil, errors.New("procfs unavailable") } - if err := CleanupMountGuards(context.Background(), newTestLogger(), false); err != nil { - t.Fatalf("CleanupMountGuards: %v", err) + if _, err := cleanupMountGuards(context.Background(), newTestLogger(), false); err != nil { + t.Fatalf("cleanupMountGuards: %v", err) } if len(*ran) != 0 { t.Fatalf("chattr -i must not run when mount status is inconclusive, calls=%#v", *ran) @@ -420,8 +420,8 @@ func TestCleanupMountGuards_SymlinkEscapeRefused(t *testing.T) { // /mnt/pve/evil passes the string allowlist but resolves outside it. resolveGuardTarget = func(string) (string, error) { return "/etc/evil", nil } - if err := CleanupMountGuards(context.Background(), newTestLogger(), false); err != nil { - t.Fatalf("CleanupMountGuards: %v", err) + if _, err := cleanupMountGuards(context.Background(), newTestLogger(), false); err != nil { + t.Fatalf("cleanupMountGuards: %v", err) } if len(*ran) != 0 { t.Fatalf("a symlink-escaping target must not be cleared, calls=%#v", *ran) @@ -442,8 +442,8 @@ func TestCleanupMountGuards_ResolveErrorLeftPending(t *testing.T) { removed := false cleanupRemoveAll = func(string) error { removed = true; return nil } - if err := CleanupMountGuards(context.Background(), newTestLogger(), false); err != nil { - t.Fatalf("CleanupMountGuards: %v", err) + if _, err := cleanupMountGuards(context.Background(), newTestLogger(), false); err != nil { + t.Fatalf("cleanupMountGuards: %v", err) } if len(*ran) != 0 { t.Fatalf("a target whose resolution errors must not be chattr'd, calls=%#v", *ran) @@ -477,8 +477,8 @@ func TestCleanupMountGuards_SymlinkedMountpointDetectedMountedAndLeftPending(t * removed := false cleanupRemoveAll = func(string) error { removed = true; return nil } - if err := CleanupMountGuards(context.Background(), newTestLogger(), false); err != nil { - t.Fatalf("CleanupMountGuards: %v", err) + if _, err := cleanupMountGuards(context.Background(), newTestLogger(), false); err != nil { + t.Fatalf("cleanupMountGuards: %v", err) } if len(*ran) != 0 { t.Fatalf("a symlinked mountpoint resolving onto a live mount must not be chattr'd, calls=%#v", *ran) @@ -498,8 +498,8 @@ func TestCleanupMountGuards_PendingKeepsIndexDir(t *testing.T) { removed := false cleanupRemoveAll = func(string) error { removed = true; return nil } - if err := CleanupMountGuards(context.Background(), newTestLogger(), false); err != nil { - t.Fatalf("CleanupMountGuards: %v", err) + if _, err := cleanupMountGuards(context.Background(), newTestLogger(), false); err != nil { + t.Fatalf("cleanupMountGuards: %v", err) } if len(*ran) != 0 { t.Fatalf("mounted target must be skipped, calls=%#v", *ran) @@ -509,7 +509,7 @@ func TestCleanupMountGuards_PendingKeepsIndexDir(t *testing.T) { } } -// End-to-end: a real recorded index is read back and cleared by CleanupMountGuards +// End-to-end: a real recorded index is read back and cleared by cleanupMountGuards // (proves record and clear agree on the on-disk format and path). func TestCleanupMountGuards_RoundTripFromRecord(t *testing.T) { withTempGuardBaseDir(t) @@ -545,8 +545,8 @@ func TestCleanupMountGuards_RoundTripFromRecord(t *testing.T) { return nil, nil } - if err := CleanupMountGuards(context.Background(), newTestLogger(), false); err != nil { - t.Fatalf("CleanupMountGuards: %v", err) + if _, err := cleanupMountGuards(context.Background(), newTestLogger(), false); err != nil { + t.Fatalf("cleanupMountGuards: %v", err) } if len(ran) != 1 || ran[0] != "chattr -i /mnt/pve/roundtrip" { t.Fatalf("round-trip: runner calls=%#v want [chattr -i /mnt/pve/roundtrip]", ran) @@ -597,8 +597,8 @@ func TestCleanupMountGuards_SummaryReported(t *testing.T) { logger := logging.New(types.LogLevelInfo, false) var buf bytes.Buffer logger.SetOutput(&buf) - if err := CleanupMountGuards(context.Background(), logger, false); err != nil { - t.Fatalf("CleanupMountGuards: %v", err) + if _, err := cleanupMountGuards(context.Background(), logger, false); err != nil { + t.Fatalf("cleanupMountGuards: %v", err) } out := buf.String() if !strings.Contains(out, "Guard cleanup summary:") { @@ -622,8 +622,8 @@ func TestCleanupMountGuards_SummaryPending(t *testing.T) { logger := logging.New(types.LogLevelInfo, false) var buf bytes.Buffer logger.SetOutput(&buf) - if err := CleanupMountGuards(context.Background(), logger, false); err != nil { - t.Fatalf("CleanupMountGuards: %v", err) + if _, err := cleanupMountGuards(context.Background(), logger, false); err != nil { + t.Fatalf("cleanupMountGuards: %v", err) } out := buf.String() if !strings.Contains(out, "immutable-cleared=0") || !strings.Contains(out, "immutable-pending=1") { From acc61b3ba571ff29aea568c1e5c71b2bbeacbb6a Mon Sep 17 00:00:00 2001 From: tis24dev Date: Mon, 3 Aug 2026 17:48:48 +0200 Subject: [PATCH 20/50] fix(install): stop both wizards fabricating an answer for EMAIL_FALLBACK_SENDMAIL The engine has always held a preserve rule for this key: an explicit answer wins, a config that stores nothing gets the failover seeded on, a config carrying only the transitional EMAIL_FALLBACK_PMF has its value migrated, and a config that already stores one keeps it. The last outcome was unreachable. Both wizards sent a hardcoded true whenever email was enabled, and neither has ever had a prompt to justify it, so the explicit-answer arm fired on every pass. An operator who set EMAIL_FALLBACK_SENDMAIL=false therefore had it flipped back to true by any wizard pass, including a no-op edit that accepts every default. The key gates the local /usr/sbin/sendmail delivery leg: with it off the delivery error is returned instead of shelling out. Flipping it silently re-opened a route someone had deliberately closed, on both front-ends. Both now leave the field nil, which is the honest statement -- there is no operator answer to forward. The field stays as the engine's seam for the day a prompt is signed off; the type doc now says a pointer means "an operator answered this", so fabricating one is visibly wrong rather than merely undocumented. Two further holes closed while the arms were being made reachable: - The arm is selected on PRESENCE in the parsed config, not on the value being non-empty. A bare "EMAIL_FALLBACK_SENDMAIL=" line is a stored OFF -- the loader stops at the first key it finds and reads the empty value as false -- but the value test read it as "nothing stored" and seeded true over it. The same trap one spelling down: an empty EMAIL_FALLBACK_PMF now migrates as false instead of seeding true. Both were the original defect wearing a different spelling. - The transitional key is retired on every arm, the preserve arm included, so a config carrying both spellings leaves with one. It stays inside the email-enabled branch: the email-off branch writes no EMAIL_FALLBACK_SENDMAIL, so retiring the alias there would flip a PMF-only config to the loader's default. The preserve arm writes nothing at all rather than canonicalising, so an operator's own spelling -- quotes, an export prefix, "no" -- survives an edit byte for byte. All three characterization goldens are untouched, and the previously pinned bug-compatibility test now pins the fixed contract instead: the firewall toggle stays non-nil because the wizard asks it, the sendmail one is nil because neither front-end does. Consequence worth stating: with no prompt on either surface, a stored false can no longer be re-enabled through the wizard. Overwrite or a hand edit are the routes back. That is the graphics freeze showing through, not an accident, and it is the argument for signing off a prompt later. Verified by mutation: 8 one-line production edits, all 8 killed by a named test, including the two that only the presence-based selection catches and the alias precedence that was previously unpinned. --- cmd/proxsave/install.go | 23 ++- cmd/proxsave/install_test.go | 99 +++++++++--- internal/installer/install_data.go | 68 ++++++-- internal/installer/install_data_test.go | 182 ++++++++++++++++++++++ internal/ui/flows/install/install.go | 10 +- internal/ui/flows/install/install_test.go | 70 ++++++++- 6 files changed, 409 insertions(+), 43 deletions(-) diff --git a/cmd/proxsave/install.go b/cmd/proxsave/install.go index f24b1131..19970f32 100644 --- a/cmd/proxsave/install.go +++ b/cmd/proxsave/install.go @@ -550,8 +550,10 @@ func runConfigWizardCLI(ctx context.Context, reader *bufio.Reader, configPath, t // Prefixing is safe because the marker only flips editingExisting, which is // UNOBSERVABLE for a CLI payload: the blank base carries no BOT_TELEGRAM_TYPE (so the // key is seeded either way), the collector always supplies a non-empty -// EmailDeliveryMethod, and EmailFallbackSendmail is always non-nil. Pinned by -// TestRunConfigWizardCLIBlankEditKeepsMinimalKeySet. +// EmailDeliveryMethod, and the blank base carries neither EMAIL_FALLBACK_SENDMAIL nor +// the transitional EMAIL_FALLBACK_PMF - parseEnvTemplate skips the marker as a +// comment, so existingValues is empty with or without it and the engine seeds the same +// default either way. Pinned by TestRunConfigWizardCLIBlankEditKeepsMinimalKeySet. const wizardBlankEditBaseMarker = "# proxsave install wizard: blank existing configuration\n" func applyInstallDataCLI(base installWizardBase, fromExisting bool, data *installer.InstallWizardData) (string, error) { @@ -643,11 +645,18 @@ func collectInstallWizardDataCLI(ctx context.Context, reader *bufio.Reader, prom } if emailEnabled { data.EmailDeliveryMethod = emailMethod - // ALWAYS non-nil true, matching what the CLI has always written and what the - // Charm front-end sends. The engine's 3-branch preserve logic stays dead for - // both; switching to preserve semantics is a separate behavior change. - fallbackSendmail := true - data.EmailFallbackSendmail = &fallbackSendmail + // EmailFallbackSendmail stays NIL, and the Charm front-end leaves it nil too: + // no wizard step on either surface asks about the local-sendmail failover, so + // there is no operator answer to forward and installer.ApplyInstallData owns + // EMAIL_FALLBACK_SENDMAIL (seed true when nothing is stored, preserve the + // stored value on an Edit). Sending a fabricated true from here rewrote a + // deliberate EMAIL_FALLBACK_SENDMAIL=false back to true on EVERY wizard pass, + // including a no-op edit, silently re-opening the /usr/sbin/sendmail delivery + // route the operator had closed. Contrast BackupFirewallRules above, which + // stays non-nil precisely BECAUSE the wizard asks that question and prefills it + // from the stored value. Pinned by + // TestCollectInstallWizardDataCLIOnlyAnsweredTogglesAreNonNil and + // TestRunConfigWizardCLIEditPreservesStoredEmailFallbackSendmail. } logging.DebugStepBootstrap(bootstrap, "install config wizard (cli)", "configuring encryption") diff --git a/cmd/proxsave/install_test.go b/cmd/proxsave/install_test.go index 73412709..518a845f 100644 --- a/cmd/proxsave/install_test.go +++ b/cmd/proxsave/install_test.go @@ -621,9 +621,10 @@ func TestPromptNotificationsEmailDefaultsToRelay(t *testing.T) { if method != "relay" { t.Fatalf("delivery method = %q, want relay", method) } - // EMAIL_FALLBACK_SENDMAIL=true is written by installer.ApplyInstallData from the - // non-nil EmailFallbackSendmail the collector always sends (pinned by - // TestCollectInstallWizardDataCLIAlwaysSendsNonNilFlags and + // EMAIL_FALLBACK_SENDMAIL=true is seeded by installer.ApplyInstallData when the + // config stores neither fallback key; the collector deliberately sends a NIL + // EmailFallbackSendmail so an Edit preserves a stored false instead (pinned by + // TestCollectInstallWizardDataCLIOnlyAnsweredTogglesAreNonNil and // internal/installer/install_data_test.go). } @@ -1087,9 +1088,10 @@ func TestRunConfigWizardCLIBlankEditKeepsMinimalKeySet(t *testing.T) { // installWizardBase exists for. It is NOT covered by the characterization goldens: // handing installer.ApplyInstallData the EXPANDED base instead of the raw one is // byte-identical on every golden scenario (measured), because the embedded template -// already carries BOT_TELEGRAM_TYPE=centralized and the collector always supplies a -// non-empty EmailDeliveryMethod and a non-nil EmailFallbackSendmail. So this is the -// only thing that keeps the split honest. +// already carries BOT_TELEGRAM_TYPE=centralized and EMAIL_FALLBACK_SENDMAIL=true (so +// the engine's seed arm on the raw base and its preserve arm on the expanded one agree +// to the byte) and the collector always supplies a non-empty EmailDeliveryMethod. So +// this is the only thing that keeps the split honest. func TestPrepareBaseTemplateRawIsEmptyOffTheEditPath(t *testing.T) { expanded := config.DefaultEnvTemplate() @@ -1169,12 +1171,24 @@ func TestPrepareBaseTemplateRawIsEmptyOffTheEditPath(t *testing.T) { }) } -// TestCollectInstallWizardDataCLIAlwaysSendsNonNilFlags pins two deliberate -// bug-compatibilities, so the follow-up commits that change them fail loudly instead -// of silently: BACKUP_FIREWALL_RULES is always written (never "keep the stored -// value"), and EMAIL_FALLBACK_SENDMAIL is always forced true when email is on -// (never installer.ApplyInstallData's 3-branch preserve logic). -func TestCollectInstallWizardDataCLIAlwaysSendsNonNilFlags(t *testing.T) { +// TestCollectInstallWizardDataCLIOnlyAnsweredTogglesAreNonNil pins the payload rule +// the CLI and the Charm front-end now share: a pointer flag is sent non-nil ONLY when +// the wizard actually asks the operator that question. +// +// BACKUP_FIREWALL_RULES is asked (and prefilled from the stored value), so it is +// always non-nil and the engine's "keep the stored value" branch stays unreachable for +// it on purpose. +// +// EMAIL_FALLBACK_SENDMAIL is asked by NEITHER front-end, so it is always nil and +// installer.ApplyInstallData owns the key. This half of the test used to pin the +// opposite - a fabricated non-nil true - as a deliberate bug-compatibility whose whole +// job was to make this commit fail loudly rather than change behaviour in silence. It +// did, and this is that deliberate update: forcing true here rewrote a stored +// EMAIL_FALLBACK_SENDMAIL=false back to true on every wizard pass, re-opening the +// local /usr/sbin/sendmail delivery route an operator had closed. The Charm mirror is +// TestCollectWizardDataLeavesEmailFallbackToTheEngine; the operator-visible half is +// TestRunConfigWizardCLIEditPreservesStoredEmailFallbackSendmail. +func TestCollectInstallWizardDataCLIOnlyAnsweredTogglesAreNonNil(t *testing.T) { promptBase := config.DefaultEnvTemplate() t.Run("email enabled", func(t *testing.T) { @@ -1188,10 +1202,10 @@ func TestCollectInstallWizardDataCLIAlwaysSendsNonNilFlags(t *testing.T) { t.Fatalf("collectInstallWizardDataCLI error: %v", err) } if data.BackupFirewallRules == nil { - t.Fatal("BackupFirewallRules must never be nil") + t.Fatal("BackupFirewallRules must never be nil (the wizard asks that question)") } - if data.EmailFallbackSendmail == nil || !*data.EmailFallbackSendmail { - t.Fatalf("EmailFallbackSendmail must be non-nil true when email is on, got %v", data.EmailFallbackSendmail) + if data.EmailFallbackSendmail != nil { + t.Fatalf("EmailFallbackSendmail must stay nil so the engine can preserve a stored value, got %v", *data.EmailFallbackSendmail) } if data.NotificationMode != "email" { t.Fatalf("NotificationMode = %q, want email", data.NotificationMode) @@ -1218,7 +1232,7 @@ func TestCollectInstallWizardDataCLIAlwaysSendsNonNilFlags(t *testing.T) { t.Fatalf("BackupFirewallRules must be non-nil true, got %v", data.BackupFirewallRules) } if data.EmailFallbackSendmail != nil { - t.Fatal("EmailFallbackSendmail must stay nil when email is off (the engine touches neither fallback key)") + t.Fatal("EmailFallbackSendmail must stay nil with email off too (the engine then touches neither fallback key)") } if data.NotificationMode != "telegram" { t.Fatalf("NotificationMode = %q, want telegram", data.NotificationMode) @@ -1226,15 +1240,60 @@ func TestCollectInstallWizardDataCLIAlwaysSendsNonNilFlags(t *testing.T) { }) } +// TestRunConfigWizardCLIEditPreservesStoredEmailFallbackSendmail is the +// operator-visible half of the contract, through the REAL wizard and a real file +// write: an operator who closed the local /usr/sbin/sendmail delivery leg with +// EMAIL_FALLBACK_SENDMAIL=false must still have it closed after a no-op edit that +// accepted every default. It fails the moment either the collector fabricates an +// answer again or the engine stops preserving. +// +// It asserts on the written bytes for that one key rather than adding a fourth +// characterization golden: the three frozen goldens all store true, so none of them +// can see this regression, and locking a new full-file golden would freeze the prompt +// transcript again for a one-key claim. +func TestRunConfigWizardCLIEditPreservesStoredEmailFallbackSendmail(t *testing.T) { + existing := setEnvValue(editedExistingConfig(), "EMAIL_FALLBACK_SENDMAIL", "false") + // A stale transitional key alongside the stored one. It is the PROOF-OF-WRITE for + // this test: the engine retires it on every arm, so its absence from the result can + // only mean the wizard actually rewrote the file. Asserting on the preserved value + // alone would pass just as well on a run that wrote NOTHING, because the seeded + // input already carries EMAIL_FALLBACK_SENDMAIL=false and an enabled email block -- + // the test would then be green while measuring nothing at all. + existing = setEnvValue(existing, "EMAIL_FALLBACK_PMF", "true") + // Same script as the EditExistingNoOp golden: choose Edit, then Enter through + // every prompt. No prompt is added or removed by this change, so the count holds. + run := runWizardCharacterization(t, existing, "2\n"+strings.Repeat("\n", 15)) + if run.err != nil { + t.Fatalf("wizard error: %v", run.err) + } + written := string(run.configData) + if strings.Contains(written, "EMAIL_FALLBACK_PMF") { + t.Fatalf("the wizard did not rewrite the file (the stale alias survived), so nothing below is measuring the fix:\n%s", written) + } + if !strings.Contains(written, "EMAIL_FALLBACK_SENDMAIL=false") { + t.Fatalf("a no-op CLI edit re-opened the local sendmail delivery route:\n%s", written) + } + if strings.Contains(written, "EMAIL_FALLBACK_SENDMAIL=true") { + t.Fatalf("EMAIL_FALLBACK_SENDMAIL was flipped back to true:\n%s", written) + } + // The stored key must also have won over the alias, which claimed the opposite. + prefill := installer.DeriveInstallWizardPrefill(written) + if !prefill.EmailEnabled || prefill.EmailDeliveryMethod != "pmf" { + t.Fatalf("expected a real email-enabled edit, got enabled=%v method=%q", prefill.EmailEnabled, prefill.EmailDeliveryMethod) + } +} + // TestApplyInstallDataCLIFeedsTheEngineTheRawBase pins the OTHER half of the // raw/expanded split: prepareBaseTemplate producing the two views is useless if the // wizard forwards the wrong one. This cannot be covered by the characterization // goldens - handing installer.ApplyInstallData the expanded base instead of the raw // one is byte-identical on every golden scenario (measured), because the embedded -// template already carries BOT_TELEGRAM_TYPE=centralized and the collector always -// supplies a non-empty EmailDeliveryMethod and a non-nil EmailFallbackSendmail. So -// the Prompt view here is deliberately a base the engine would treat very -// differently, making the mistake visible. +// template already carries BOT_TELEGRAM_TYPE=centralized and +// EMAIL_FALLBACK_SENDMAIL=true (so the engine's seed and preserve arms agree to the +// byte) and the collector always supplies a non-empty EmailDeliveryMethod. So the +// Prompt view here is deliberately a base the engine would treat very differently, +// making the mistake visible. Its payload keeps an EXPLICIT EmailFallbackSendmail, +// unlike the collector's, so the engine's explicit-answer arm stays exercised. func TestApplyInstallDataCLIFeedsTheEngineTheRawBase(t *testing.T) { fallbackSendmail := true firewall := false diff --git a/internal/installer/install_data.go b/internal/installer/install_data.go index eb6639a3..9dac3e1b 100644 --- a/internal/installer/install_data.go +++ b/internal/installer/install_data.go @@ -53,7 +53,16 @@ type InstallWizardPrefill struct { HealthcheckMode string // "off" | "centralized" | "self" (empty on a fresh/pre-daemon config) } -// InstallWizardData holds the collected installation data +// InstallWizardData holds the collected installation data. +// +// A pointer field here means "an operator answered this question", so it is only ever +// non-nil when a front-end actually asked. EmailFallbackSendmail is the exception that +// proves the rule: NEITHER wizard has a sendmail-failover prompt, so both leave it nil +// and ApplyInstallData owns EMAIL_FALLBACK_SENDMAIL end to end (seed / migrate / +// preserve). The field is kept as the engine's UI-agnostic seam for the day such a +// prompt is signed off - it is not an invitation to fabricate a value, which is what +// used to flip a stored false back to true on every wizard pass and re-open the local +// /usr/sbin/sendmail delivery route. type InstallWizardData struct { BaseDir string ConfigPath string @@ -66,7 +75,7 @@ type InstallWizardData struct { BackupFirewallRules *bool NotificationMode string // "none", "telegram", "email", "both" EmailDeliveryMethod string // "relay", "sendmail", or "pmf" - EmailFallbackSendmail *bool + EmailFallbackSendmail *bool // nil from BOTH front-ends on purpose - see the type doc CronTime string // HH:MM (the "Run at" time) EnableEncryption bool SchedulerMode string // "cron" | "daemon" @@ -251,17 +260,58 @@ func ApplyInstallData(baseTemplate string, data *InstallWizardData) (string, err method = installEmailDeliveryMethodOrDefault(method) template = setEnvValue(template, "EMAIL_DELIVERY_METHOD", method) - fallbackRaw := readTemplateString(existingValues, "EMAIL_FALLBACK_SENDMAIL", "EMAIL_FALLBACK_PMF") + // EMAIL_FALLBACK_SENDMAIL gates the local /usr/sbin/sendmail delivery leg: + // with it off, notify.sendPMFFallbackChain returns the delivery error instead + // of shelling out, and a network-less run disables email instead of rerouting + // it through sendmail (cmd/proxsave/main_network.go). NEITHER front-end asks + // the operator about that key, so a nil payload field is the NORMAL case and + // this block owns the value end to end: + // + // explicit answer -> write it (the seam a future prompt plugs into) + // EMAIL_FALLBACK_SENDMAIL + // present at all -> PRESERVE it, byte for byte + // only the legacy + // EMAIL_FALLBACK_PMF -> migrate its value onto the current spelling + // neither key present -> seed true (a blank Edit; the shipped template + // already carries the key, so a fresh install and + // an Overwrite both take the preserve arm) + // + // The preserve arm is the point of the whole switch. Without it, a wizard pass + // that asked no question - a no-op edit included - rewrote a deliberate + // EMAIL_FALLBACK_SENDMAIL=false back to true and silently re-opened a delivery + // route the operator had closed. + // + // Every arm of this switch drops the transitional key, so a config that carried + // both spellings leaves with one. That is safe because the arms read + // existingValues, parsed from the UNTOUCHED baseTemplate, so the migrate arm + // still has the value after the line is gone from template. It is not lossless + // in general -- on the migrate arm the alias IS the key the loader was reading + // -- which is exactly why that arm copies the value across before it is lost. + // The drop stays INSIDE the email-enabled branch on purpose: the email-off + // branch writes no EMAIL_FALLBACK_SENDMAIL, so removing the alias there would + // flip a PMF-only config to the loader's true default. + // + // PRESENCE in existingValues, not emptiness of the value, is what selects the + // arm. The two are different: parseEnvTemplate records a bare "KEY=" line as a + // present, empty value, and the loader reads that as FALSE + // (getBoolWithFallback stops at the first key it finds; utils.ParseBool("") is + // false). Keying off the value instead treated "KEY=" as "nothing stored" and + // seeded true over it -- the same silent re-open this switch exists to prevent, + // just spelled with an empty value instead of "false". + _, sendmailStored := existingValues["EMAIL_FALLBACK_SENDMAIL"] + legacyRaw, legacyStored := existingValues["EMAIL_FALLBACK_PMF"] + template = unsetEnvValue(template, "EMAIL_FALLBACK_PMF") switch { case data.EmailFallbackSendmail != nil: - template = unsetEnvValue(template, "EMAIL_FALLBACK_PMF") template = setEnvValue(template, "EMAIL_FALLBACK_SENDMAIL", fmt.Sprintf("%t", *data.EmailFallbackSendmail)) - case fallbackRaw == "": - template = unsetEnvValue(template, "EMAIL_FALLBACK_PMF") + case sendmailStored: + // PRESERVE: write nothing at all. Not even a canonicalising rewrite -- the + // operator's own spelling (quotes, an export prefix, yes/on/0) survives an + // edit verbatim, and utils.ParseBool and the loader accept all of them. + case legacyStored: + template = setEnvValue(template, "EMAIL_FALLBACK_SENDMAIL", fmt.Sprintf("%t", utils.ParseBool(legacyRaw))) + default: template = setEnvValue(template, "EMAIL_FALLBACK_SENDMAIL", "true") - case strings.TrimSpace(existingValues["EMAIL_FALLBACK_SENDMAIL"]) == "": - template = unsetEnvValue(template, "EMAIL_FALLBACK_PMF") - template = setEnvValue(template, "EMAIL_FALLBACK_SENDMAIL", fmt.Sprintf("%t", utils.ParseBool(fallbackRaw))) } } else { template = setEnvValue(template, "EMAIL_ENABLED", "false") diff --git a/internal/installer/install_data_test.go b/internal/installer/install_data_test.go index 8a41ba35..343e70a2 100644 --- a/internal/installer/install_data_test.go +++ b/internal/installer/install_data_test.go @@ -294,3 +294,185 @@ func TestApplyInstallDataPreservesExistingEmailDeliveryMethod(t *testing.T) { t.Fatalf("expected transitional EMAIL_FALLBACK_PMF key to be removed:\n%s", result) } } + +// TestApplyInstallDataEmailFallbackSendmail pins all four outcomes of the key that +// gates the local /usr/sbin/sendmail delivery leg, plus the retirement of its +// transitional EMAIL_FALLBACK_PMF spelling. +// +// The load-bearing one is preserve: neither front-end asks the operator about this +// key, so both send a nil pointer, and anything other than "keep what is stored" means +// a wizard pass that asked no question re-opens a delivery route the operator closed +// on purpose. Preserve here means the stored LINE is not rewritten at all, which is +// why the fixtures below use spellings a canonicalizing rewrite would visibly change. +func TestApplyInstallDataEmailFallbackSendmail(t *testing.T) { + emailOn := func() *InstallWizardData { + return &InstallWizardData{BaseDir: "/data", NotificationMode: "email"} + } + boolPtr := func(v bool) *bool { return &v } + + t.Run("fresh install ends with the fallback on", func(t *testing.T) { + // A fresh install / Overwrite passes an EMPTY base: existingValues stays empty + // (ApplyInstallData only parses a non-blank base) while template becomes the + // embedded default, which already ships EMAIL_FALLBACK_SENDMAIL=true. + // + // So this pins the END STATE, not the arm that produced it -- the assertion + // holds whether the seed arm ran or the shipped line was simply left alone. + // The seed ARM is pinned by the sibling subtest below, which uses a base that + // carries neither key. Kept because a change to the shipped template that + // dropped the key would still be caught here. + result, err := ApplyInstallData("", emailOn()) + if err != nil { + t.Fatalf("ApplyInstallData: %v", err) + } + if !strings.Contains(result, "EMAIL_FALLBACK_SENDMAIL=true") { + t.Fatalf("fresh install must end with the fallback on:\n%s", result) + } + }) + + t.Run("edit without either fallback key seeds true", func(t *testing.T) { + // The engine side of the CLI blank-Edit marker path: editingExisting is true + // but the config predates both spellings. This is the ONLY subtest that + // reaches the seed arm with a template that does not already carry the key, + // so it is the one that dies if the arm is deleted. + result, err := ApplyInstallData("# proxsave install wizard: blank existing configuration\nEMAIL_ENABLED=false\n", emailOn()) + if err != nil { + t.Fatalf("ApplyInstallData: %v", err) + } + if !strings.Contains(result, "EMAIL_FALLBACK_SENDMAIL=true") { + t.Fatalf("an edit with nothing stored must seed the fallback on:\n%s", result) + } + }) + + t.Run("a present but empty value is preserved, not seeded over", func(t *testing.T) { + // "KEY=" is a stored OFF, not an absent key: the loader reads it as false + // (getBoolWithFallback stops at the first key it FINDS, and ParseBool("") is + // false). Selecting the arm on the VALUE rather than on presence treated this + // as "nothing stored" and seeded true over it -- the same silent re-open of + // the sendmail route as the "false" case, spelled differently. + result, err := ApplyInstallData("EMAIL_ENABLED=true\nEMAIL_FALLBACK_SENDMAIL=\n", emailOn()) + if err != nil { + t.Fatalf("ApplyInstallData: %v", err) + } + if strings.Contains(result, "EMAIL_FALLBACK_SENDMAIL=true") { + t.Fatalf("a bare KEY= is a stored off; a no-op edit must not turn it on:\n%s", result) + } + }) + + t.Run("a present but empty legacy key migrates as off", func(t *testing.T) { + // Same trap one spelling down. The loader reads this config as false, so the + // migration must carry that across rather than read the empty value as + // "nothing stored" and seed true. + result, err := ApplyInstallData("EMAIL_ENABLED=true\nEMAIL_FALLBACK_PMF=\n", emailOn()) + if err != nil { + t.Fatalf("ApplyInstallData: %v", err) + } + if !strings.Contains(result, "EMAIL_FALLBACK_SENDMAIL=false") { + t.Fatalf("an empty legacy key means off and must migrate as off:\n%s", result) + } + if strings.Contains(result, "EMAIL_FALLBACK_PMF") { + t.Fatalf("the transitional key must be retired by the migration:\n%s", result) + } + }) + + t.Run("stored value is preserved verbatim", func(t *testing.T) { + // Non-canonical on purpose: an export prefix, quotes, a value ParseBool reads + // as false and an inline comment. All of it must survive an edit untouched. + stored := `export EMAIL_FALLBACK_SENDMAIL="no" # closed deliberately` + result, err := ApplyInstallData("EMAIL_ENABLED=true\n"+stored+"\n", emailOn()) + if err != nil { + t.Fatalf("ApplyInstallData: %v", err) + } + if !strings.Contains(result, stored) { + t.Fatalf("the stored fallback line must survive byte-identical, want %q in:\n%s", stored, result) + } + if strings.Contains(result, "EMAIL_FALLBACK_SENDMAIL=true") { + t.Fatalf("a no-op edit must not re-open the sendmail route:\n%s", result) + } + }) + + t.Run("stored false is preserved", func(t *testing.T) { + result, err := ApplyInstallData("EMAIL_ENABLED=true\nEMAIL_DELIVERY_METHOD=relay\nEMAIL_FALLBACK_SENDMAIL=false\n", emailOn()) + if err != nil { + t.Fatalf("ApplyInstallData: %v", err) + } + if !strings.Contains(result, "EMAIL_FALLBACK_SENDMAIL=false") { + t.Fatalf("a stored false must survive an edit:\n%s", result) + } + if strings.Contains(result, "EMAIL_FALLBACK_SENDMAIL=true") { + t.Fatalf("a no-op edit must not re-open the sendmail route:\n%s", result) + } + }) + + t.Run("legacy pmf value is migrated", func(t *testing.T) { + // "no" rather than "false" so the migrated VALUE is pinned, not just its + // presence: writing a literal true here would destroy the operator's choice. + result, err := ApplyInstallData("EMAIL_ENABLED=true\nEMAIL_FALLBACK_PMF=no\n", emailOn()) + if err != nil { + t.Fatalf("ApplyInstallData: %v", err) + } + if !strings.Contains(result, "EMAIL_FALLBACK_SENDMAIL=false") { + t.Fatalf("the legacy value must be migrated onto the current key:\n%s", result) + } + if strings.Contains(result, "EMAIL_FALLBACK_PMF") { + t.Fatalf("the transitional key must be retired by the migration:\n%s", result) + } + }) + + t.Run("stale pmf is dropped while the stored value is preserved", func(t *testing.T) { + // Both spellings present. config.getBoolWithFallback reads + // EMAIL_FALLBACK_SENDMAIL first, so the engine must agree on that precedence + // (preserve false, not adopt the alias's true) and must not leave the config + // carrying two sources of truth. + result, err := ApplyInstallData("EMAIL_ENABLED=true\nEMAIL_FALLBACK_SENDMAIL=false\nEMAIL_FALLBACK_PMF=true\n", emailOn()) + if err != nil { + t.Fatalf("ApplyInstallData: %v", err) + } + if !strings.Contains(result, "EMAIL_FALLBACK_SENDMAIL=false") { + t.Fatalf("the stored key must win over the transitional alias:\n%s", result) + } + if strings.Contains(result, "EMAIL_FALLBACK_PMF") { + t.Fatalf("the transitional key must be dropped on the preserve path too:\n%s", result) + } + }) + + t.Run("explicit false overrides a stored true", func(t *testing.T) { + data := emailOn() + data.EmailFallbackSendmail = boolPtr(false) + result, err := ApplyInstallData("EMAIL_ENABLED=true\nEMAIL_FALLBACK_SENDMAIL=true\n", data) + if err != nil { + t.Fatalf("ApplyInstallData: %v", err) + } + if !strings.Contains(result, "EMAIL_FALLBACK_SENDMAIL=false") { + t.Fatalf("an explicit answer must beat the stored value:\n%s", result) + } + }) + + t.Run("explicit true overrides a stored false", func(t *testing.T) { + // No front-end sends this today; the arm exists so the engine stays the one + // place that can be told, for the day a prompt is signed off. + data := emailOn() + data.EmailFallbackSendmail = boolPtr(true) + result, err := ApplyInstallData("EMAIL_ENABLED=true\nEMAIL_FALLBACK_SENDMAIL=false\n", data) + if err != nil { + t.Fatalf("ApplyInstallData: %v", err) + } + if !strings.Contains(result, "EMAIL_FALLBACK_SENDMAIL=true") { + t.Fatalf("an explicit answer must beat the stored value:\n%s", result) + } + }) + + t.Run("email disabled touches neither key", func(t *testing.T) { + // The scope line: with email off the engine writes no + // EMAIL_FALLBACK_SENDMAIL, so retiring the alias here would silently flip a + // PMF-only config to the loader's true default. + base := "EMAIL_ENABLED=true\nEMAIL_FALLBACK_SENDMAIL=false\nEMAIL_FALLBACK_PMF=true\n" + data := &InstallWizardData{BaseDir: "/data", NotificationMode: "none"} + result, err := ApplyInstallData(base, data) + if err != nil { + t.Fatalf("ApplyInstallData: %v", err) + } + if !strings.Contains(result, "EMAIL_FALLBACK_SENDMAIL=false") || !strings.Contains(result, "EMAIL_FALLBACK_PMF=true") { + t.Fatalf("with email off both fallback keys must survive verbatim:\n%s", result) + } + }) +} diff --git a/internal/ui/flows/install/install.go b/internal/ui/flows/install/install.go index cc3c1c0c..305d2cf9 100644 --- a/internal/ui/flows/install/install.go +++ b/internal/ui/flows/install/install.go @@ -265,8 +265,14 @@ func CollectWizardData(ctx context.Context, session *shell.Session, baseTemplate } if email.Bool { data.EmailDeliveryMethod = methodValues[method.OptionIndex] - fallbackSendmail := true - data.EmailFallbackSendmail = &fallbackSendmail + // EmailFallbackSendmail stays NIL, the same contract the CLI wizard follows + // (cmd/proxsave/install.go): this form has no sendmail-failover row, so there + // is no operator answer to send and installer.ApplyInstallData owns + // EMAIL_FALLBACK_SENDMAIL (seed true when nothing is stored, preserve the + // stored value on an Edit). A fabricated true here re-opened the local + // /usr/sbin/sendmail delivery route on every no-op edit of a config that had + // deliberately closed it. Pinned by + // TestCollectWizardDataLeavesEmailFallbackToTheEngine. } normalized, err := cronutil.NormalizeTime(cronField.Text, cronutil.DefaultTime) if err != nil { diff --git a/internal/ui/flows/install/install_test.go b/internal/ui/flows/install/install_test.go index b8e9edee..d370bec5 100644 --- a/internal/ui/flows/install/install_test.go +++ b/internal/ui/flows/install/install_test.go @@ -187,11 +187,12 @@ func TestCollectWizardDataDeclineAll(t *testing.T) { } } -// TestCollectWizardDataPrefillNoOp locks the anti-drift core: with a fully -// populated existing template, an Enter-only run returns exactly the stored -// settings (the historical no-op-edit reset bug). -func TestCollectWizardDataPrefillNoOp(t *testing.T) { - d := newDriver(t) +// prefilledEditTemplate is the fully-populated existing config the no-op-edit tests +// share: every toggle on, so all 13 form rows are active (healthchecks stay dimmed +// because the template ships SCHEDULER_MODE=cron) and an Enter-only run is exactly 14 +// key presses. Mirrors editedExistingConfig() on the CLI side, so both front-ends are +// driven through the same no-op-edit gesture. +func prefilledEditTemplate() string { template := config.DefaultEnvTemplate() for _, kv := range [][2]string{ {"SECONDARY_ENABLED", "true"}, @@ -209,6 +210,15 @@ func TestCollectWizardDataPrefillNoOp(t *testing.T) { } { template = installer.SetEnvValueInTemplate(template, kv[0], kv[1]) } + return template +} + +// TestCollectWizardDataPrefillNoOp locks the anti-drift core: with a fully +// populated existing template, an Enter-only run returns exactly the stored +// settings (the historical no-op-edit reset bug). +func TestCollectWizardDataPrefillNoOp(t *testing.T) { + d := newDriver(t) + template := prefilledEditTemplate() type result struct { data *installer.InstallWizardData @@ -263,6 +273,56 @@ func TestCollectWizardDataPrefillNoOp(t *testing.T) { } } +// TestCollectWizardDataLeavesEmailFallbackToTheEngine is the dashboard half of a +// contract the CLI wizard also has to keep: neither front-end asks the operator about +// the local /usr/sbin/sendmail failover, so neither may fabricate an answer for it. +// The form sends a nil EmailFallbackSendmail and installer.ApplyInstallData owns +// EMAIL_FALLBACK_SENDMAIL, which is what lets a stored false survive a no-op edit +// instead of being flipped back to true and silently re-opening that delivery route. +// +// This is the only test standing between the dashboard and a re-introduced hardcoded +// true: this package has no byte-level characterization golden, so the CLI's goldens +// and end-to-end tests cannot see a Charm-side regression at all. Repairing one +// front-end is not a repair. +func TestCollectWizardDataLeavesEmailFallbackToTheEngine(t *testing.T) { + d := newDriver(t) + template := installer.SetEnvValueInTemplate(prefilledEditTemplate(), "EMAIL_FALLBACK_SENDMAIL", "false") + + resCh := collectWizardAsync(t, d, template) + + // Same no-op-edit gesture as TestCollectWizardDataPrefillNoOp: 13 active rows + // plus Continue. No form row is added or removed by leaving the field nil. + d.waitScreen("Configuration") + for i := 0; i < 14; i++ { + d.keys("enter") + } + + res := <-resCh + if res.err != nil { + t.Fatalf("unexpected error: %v", res.err) + } + if res.data.EmailFallbackSendmail != nil { + t.Fatalf("EmailFallbackSendmail must stay nil so the engine can preserve the stored value, got %v", *res.data.EmailFallbackSendmail) + } + // Guard against a vacuous pass: the email branch must actually have run. + if res.data.NotificationMode != "both" || res.data.EmailDeliveryMethod != "pmf" { + t.Fatalf("expected a real email-enabled edit, got mode=%q method=%q", res.data.NotificationMode, res.data.EmailDeliveryMethod) + } + + // End to end through the shared engine: nil collector + preserve branch must + // compose into a config that still has the sendmail route closed. + out, err := installer.ApplyInstallData(template, res.data) + if err != nil { + t.Fatalf("ApplyInstallData: %v", err) + } + if !strings.Contains(out, "EMAIL_FALLBACK_SENDMAIL=false") { + t.Fatalf("a no-op dashboard edit re-opened the local sendmail delivery route:\n%s", out) + } + if strings.Contains(out, "EMAIL_FALLBACK_SENDMAIL=true") { + t.Fatalf("EMAIL_FALLBACK_SENDMAIL was flipped back to true:\n%s", out) + } +} + // TestCollectWizardDataEditWithoutSchedulerModeDefaultsCron locks the no-op-edit // invariant for a legacy/pre-daemon config that lacks SCHEDULER_MODE: an Enter-only // edit must NOT silently flip the scheduler to daemon (it stays on cron, matching From fd66954395451f2817a50692d4a903875cbff293 Mon Sep 17 00:00:00 2001 From: tis24dev Date: Mon, 3 Aug 2026 17:56:58 +0200 Subject: [PATCH 21/50] fix(install): stop offering the template's example rclone remote as a default Off the Edit path prepareBaseTemplate hands the CLI wizard the SHIPPED template, so every "stored value" the prefill reads back is really a template line. That is harmless for the keys the template leaves blank (SECONDARY_PATH) or seeds with a usable default (SCHEDULER_TIME=02:00) -- but CLOUD_REMOTE ships as an EXAMPLE remote NAME, "GoogleDrive", which only works on a host that happens to have named its rclone remote exactly that. Offering it as the prompt default meant an operator who enabled cloud backups and accepted the defaults finished with CLOUD_ENABLED=true pointed at a remote rclone has never heard of. Because a cloud upload failure is deliberately non-critical, that does not fail the install: it surfaces as a warning on every run from then on, which is the hardest kind of misconfiguration to trace back to its cause. The default is now dropped on that path, so the prompt requires an answer. This is the same defence schedulerEngineDefault, healthcheckModeDefault and cronTimeDefault already apply to their own keys -- CLOUD_REMOTE was the one that was missed. CLOUD_LOG_PATH is deliberately left alone. "/proxsave/log" is a real path inside whatever remote is chosen and the template's own comment lists it as an accepted form, so it is a usable default rather than a stand-in. A mutation that clears it too is caught. Scope: the dashboard is unaffected. It passes an empty base on a fresh install and never expands the template, so it has never shown these values -- the comment on prepareBaseTemplate already recorded that asymmetry ("unlike the Charm one"). One golden line moves, deliberately: fresh_enable_all.transcript loses the "[GoogleDrive] " default. No .env golden moves, so the bytes written to a config are unchanged. Behaviour note for scripted installs: an answer stream that relied on pressing Enter here now hits a required prompt and, if finite, ends at EOF rather than silently configuring a remote that does not exist. Verified by mutation: 3 one-line edits, all 3 killed by a named test, including the over-correction that would drop the log default as well. --- cmd/proxsave/install.go | 21 +++ cmd/proxsave/install_cloud_prefill_test.go | 125 ++++++++++++++++++ .../fresh_enable_all.transcript | 2 +- 3 files changed, 147 insertions(+), 1 deletion(-) create mode 100644 cmd/proxsave/install_cloud_prefill_test.go diff --git a/cmd/proxsave/install.go b/cmd/proxsave/install.go index 19970f32..b0de98e6 100644 --- a/cmd/proxsave/install.go +++ b/cmd/proxsave/install.go @@ -598,6 +598,27 @@ func collectInstallWizardDataCLI(ctx context.Context, reader *bufio.Reader, prom prefillBase = wizardBlankBaseStandIn } prefill := installer.DeriveInstallWizardPrefill(prefillBase) + if !fromExisting { + // Off the Edit path prefillBase is the SHIPPED template (prepareBaseTemplate + // expands it there), so every "stored value" read out of it is really a + // template line. That is harmless for the keys the template leaves blank or + // seeds with a usable default - but CLOUD_REMOTE ships as an EXAMPLE remote + // NAME, "GoogleDrive", which only works on a host that happens to have named + // its rclone remote exactly that. Offering it as the prompt default meant + // pressing Enter wrote CLOUD_ENABLED=true against a remote that does not + // exist, and the operator then got a warning on every run instead of a + // failure at install time, because a cloud upload failure is non-critical. + // + // Dropping the default degrades the prompt to "no default", which + // promptNonEmptyWithDefault turns into a required answer. That is the same + // defence schedulerEngineDefault / healthcheckModeDefault / cronTimeDefault + // already apply to their keys; this one was missed. + // + // CLOUD_LOG_PATH is deliberately NOT cleared: "/proxsave/log" is a real path + // inside whatever remote is chosen, listed by the template's own comment as an + // accepted form, so it is a usable default rather than a stand-in. + prefill.CloudRemote = "" + } data := &installer.InstallWizardData{} diff --git a/cmd/proxsave/install_cloud_prefill_test.go b/cmd/proxsave/install_cloud_prefill_test.go new file mode 100644 index 00000000..b3d53193 --- /dev/null +++ b/cmd/proxsave/install_cloud_prefill_test.go @@ -0,0 +1,125 @@ +package main + +import ( + "bufio" + "context" + "strings" + "testing" + + "github.com/tis24dev/proxsave/internal/config" + "github.com/tis24dev/proxsave/internal/installer" +) + +// templateCloudRemoteExample is the value the shipped template seeds CLOUD_REMOTE with. +// It is read out of the template rather than written here so the test keeps pointing at +// the real example if it is ever reworded, instead of silently passing against a literal +// nobody ships any more. +func templateCloudRemoteExample(t *testing.T) string { + t.Helper() + example := strings.TrimSpace(installer.DeriveInstallWizardPrefill(config.DefaultEnvTemplate()).CloudRemote) + if example == "" { + t.Skip("the shipped template no longer seeds CLOUD_REMOTE; this trap cannot fire") + } + return example +} + +// TestFreshInstallDoesNotOfferTheTemplateCloudRemote pins the fix for a defect that made +// pressing Enter enough to finish an install pointed at a remote that does not exist. +// +// Off the Edit path prepareBaseTemplate hands the wizard the SHIPPED template, so every +// value the prefill reads back is a template line rather than an operator's choice. Most +// of those are blank or a usable default, but CLOUD_REMOTE ships as an EXAMPLE remote +// NAME. Offering it as the prompt default meant an operator who accepted the defaults got +// CLOUD_ENABLED=true against a remote rclone has never heard of — and because a cloud +// upload failure is deliberately non-critical, that surfaces as a warning on every run +// from then on rather than as a failure at install time. +// +// The prompt must therefore REQUIRE an answer here. schedulerEngineDefault, +// healthcheckModeDefault and cronTimeDefault already apply the same rule to their keys. +func TestFreshInstallDoesNotOfferTheTemplateCloudRemote(t *testing.T) { + example := templateCloudRemoteExample(t) + + // Enable cloud, then answer the two rclone prompts. Everything after is defaulted. + script := strings.Join([]string{ + "n", // secondary + "y", // cloud + "myremote:pbs-backups", // remote (now REQUIRED: no default is offered) + "myremote:/logs", // log remote + "n", // firewall + "n", // telegram + "n", // email + "n", // encryption + "", // scheduler engine: default + "off", // healthchecks + "", // run at: default + }, "\n") + "\n" + + var data *installer.InstallWizardData + var err error + transcript := captureStdout(t, func() { + reader := bufio.NewReader(strings.NewReader(script)) + data, err = collectInstallWizardDataCLI(context.Background(), reader, config.DefaultEnvTemplate(), false, nil) + }) + if err != nil { + t.Fatalf("collectInstallWizardDataCLI error: %v", err) + } + if strings.Contains(transcript, example) { + t.Fatalf("a fresh install offered the template's example remote %q as a default:\n%s", example, transcript) + } + if strings.Contains(transcript, "["+example+"]") { + t.Fatalf("the example remote is still rendered as a prompt default:\n%s", transcript) + } + // Guard against a vacuous pass: the cloud branch must really have run, and the + // typed answer — not a default — must be what reached the payload. + if !data.EnableCloudStorage { + t.Fatal("the cloud branch did not run, so nothing above measured the prompt") + } + if data.RcloneBackupRemote != "myremote:pbs-backups" { + t.Fatalf("the typed remote must reach the payload, got %q", data.RcloneBackupRemote) + } + // The log path default is deliberately NOT dropped: it is a real path inside + // whatever remote is chosen, which the template's own comment lists as an accepted + // form. Losing it here would be an unrelated regression. + if !strings.Contains(transcript, "Rclone remote for logs") { + t.Fatalf("the log prompt is missing entirely:\n%s", transcript) + } +} + +// TestEditStillOffersTheStoredCloudRemote is the other half of the contract: dropping the +// default must be scoped to the path where the "stored" value is really a template line. +// On an Edit the value belongs to the operator, and a no-op edit has to keep it — that is +// the whole point of prefilling. +func TestEditStillOffersTheStoredCloudRemote(t *testing.T) { + stored := "operator-remote:archive" + base := "CLOUD_ENABLED=true\nCLOUD_REMOTE=" + stored + "\nCLOUD_LOG_PATH=/proxsave/log\n" + + script := strings.Join([]string{ + "n", // secondary + "", // cloud: keep enabled (prefilled) + "", // remote: accept the STORED value + "", // log remote: accept the stored value + "n", // firewall + "n", // telegram + "n", // email + "n", // encryption + "", // scheduler engine + "off", + "", // run at + }, "\n") + "\n" + + var data *installer.InstallWizardData + var err error + transcript := captureStdout(t, func() { + reader := bufio.NewReader(strings.NewReader(script)) + data, err = collectInstallWizardDataCLI(context.Background(), reader, base, true, nil) + }) + if err != nil { + t.Fatalf("collectInstallWizardDataCLI error: %v", err) + } + if !strings.Contains(transcript, "["+stored+"]") { + t.Fatalf("an edit must still offer the stored remote as the default:\n%s", transcript) + } + if data.RcloneBackupRemote != stored { + t.Fatalf("a no-op edit must keep the stored remote, got %q", data.RcloneBackupRemote) + } +} diff --git a/cmd/proxsave/testdata/install_characterization/fresh_enable_all.transcript b/cmd/proxsave/testdata/install_characterization/fresh_enable_all.transcript index f0d1ef3d..c57b6ebe 100644 --- a/cmd/proxsave/testdata/install_characterization/fresh_enable_all.transcript +++ b/cmd/proxsave/testdata/install_characterization/fresh_enable_all.transcript @@ -8,7 +8,7 @@ For direct network access without mounting, use cloud storage (rclone) instead. Enable secondary backup path? [y/N]: Secondary backup path (SECONDARY_PATH): Secondary log path (SECONDARY_LOG_PATH, optional - press Enter to skip): --- Cloud storage (rclone) --- Remember to configure rclone manually before enabling cloud backups. -Enable cloud backups? [y/N]: Rclone remote for backups (e.g. myremote:pbs-backups): [GoogleDrive] Rclone remote for logs (e.g. myremote:/logs): [/proxsave/log] +Enable cloud backups? [y/N]: Rclone remote for backups (e.g. myremote:pbs-backups): Rclone remote for logs (e.g. myremote:/logs): [/proxsave/log] --- Firewall rules --- Enable collection of firewall rules (e.g., iptables/nftables). (You can change this later in backup.env via BACKUP_FIREWALL_RULES) From 67ce0ff5c9a5c40f26c3d9fb68a1ffcc50025202 Mon Sep 17 00:00:00 2001 From: tis24dev Date: Mon, 3 Aug 2026 18:25:35 +0200 Subject: [PATCH 22/50] docs(install): correct two false comments about the firewall-rules nil arm MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both were found by a four-angle refutation pass aimed at the claim that the arm is dead code. The claim survived — compiler enumeration via an overlay type rename finds exactly two production constructors, and instrumenting ApplyInstallData over the whole suite logs 34 nil arrivals, every one of them from a unit-test body with no production frame in the stack. But two comments around it are wrong. install.go said "the CLI has ALWAYS written BACKUP_FIREWALL_RULES on every run". It has not: the CLI wizard existed for over a month without that key, which only appeared in 2026-01. Worse as an argument, the CLI did not go through the engine at all until the commit that added this very assignment, so "always" was resting on a shape of the code that was days old. It now says what is actually load-bearing — this wizard asks the question, so it always has an answer — and says plainly that unreachable is not the same as guaranteed. install_data.go said the nil arm "keeps the template default when unset". That names only half of it. A nil field means the key is not written at all, and what that leaves behind differs by path: the SHIPPED template's value on a fresh install or an Overwrite, the OPERATOR's stored value on an Edit. Those are not the same thing, and on the Edit path calling it a template default is simply wrong. Both comments now also record why the guard stays despite being unreachable: the property "both front-ends send non-nil" is not an invariant. The sibling field EmailFallbackSendmail went from always-non-nil to always-nil on BOTH surfaces one commit ago, which is the same mutation applied to the same shape. And the nil check is what keeps this exported function from dereferencing a nil pointer — removing it crashes 29 call sites across 11 test functions. Comment-only: no statement or expression changed. --- cmd/proxsave/install.go | 12 ++++++++++-- internal/installer/install_data.go | 14 +++++++++++++- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/cmd/proxsave/install.go b/cmd/proxsave/install.go index b0de98e6..c29d87b5 100644 --- a/cmd/proxsave/install.go +++ b/cmd/proxsave/install.go @@ -645,8 +645,16 @@ func collectInstallWizardDataCLI(ctx context.Context, reader *bufio.Reader, prom if err != nil { return nil, err } - // ALWAYS non-nil: the CLI has always written BACKUP_FIREWALL_RULES on every run, - // so the engine's nil "keep the stored value" branch must stay unreachable here. + // Unconditional, and it must stay that way: this wizard ASKS the question, so it + // always has an answer to forward and the engine's nil arm is unreachable from + // here. Contrast EmailFallbackSendmail below, which is left nil precisely because + // nothing asks. + // + // Do not read "unreachable" as "guaranteed". The engine's nil arm being dead is a + // property of today's two front-ends, not an invariant: this assignment only + // entered the tree when the CLI wizard started going through the engine at all, + // and the sibling field moved the other way (non-nil to nil, both front-ends) in + // the very next campaign item. The guard on the engine side stays. data.BackupFirewallRules = &firewallEnabled logging.DebugStepBootstrap(bootstrap, "install config wizard (cli)", "configuring notifications") diff --git a/internal/installer/install_data.go b/internal/installer/install_data.go index 9dac3e1b..1691d1f6 100644 --- a/internal/installer/install_data.go +++ b/internal/installer/install_data.go @@ -231,7 +231,19 @@ func ApplyInstallData(baseTemplate string, data *InstallWizardData) (string, err template = setEnvValue(template, "CLOUD_LOG_PATH", "") } - // Apply firewall rules backup (optional; keep template default when unset) + // Apply firewall rules backup. A nil field means no front-end asked, and the key is + // then not written AT ALL -- which is not the same thing on both paths: on a fresh + // install or an Overwrite it leaves the SHIPPED template's value, on an Edit it + // leaves the OPERATOR's stored value. Calling that "the template default" names + // only the first. + // + // Both wizards ask the firewall question today, so the nil arm is currently + // unreachable from production (verified by instrumenting this function over the + // whole test suite: every nil arrival comes from a unit-test body, never from a + // production frame). It is kept because that is a property of today's front-ends + // rather than an invariant -- the sibling EmailFallbackSendmail went from + // always-non-nil to always-nil on both surfaces -- and because the nil check is + // what stops this exported function dereferencing a nil pointer. if data.BackupFirewallRules != nil { if *data.BackupFirewallRules { template = setEnvValue(template, "BACKUP_FIREWALL_RULES", "true") From 4f4f75d746e44f434c5c5d3a34caf95eb0505998 Mon Sep 17 00:00:00 2001 From: tis24dev Date: Mon, 3 Aug 2026 19:55:36 +0200 Subject: [PATCH 23/50] test: pin the firewall answer against the stored value, and the loader default against the template Two small gaps, both on code production actually runs. They came out of a design pass whose main conclusion was NOT to do what it was asked: characterizing the BackupFirewallRules nil arm was measured and rejected, because that arm is unreachable and nine of ten plausible subtests for it could not fail. The first gap is the direction no golden covers: an operator CHANGING the firewall answer on an edit. All three characterization goldens agree with whatever was already stored -- the two fresh ones store nothing, edit_noop answers true over a stored true -- so a one-line change that makes the stored value win over the answered one passed the entire repo. The failure that hides is silent and total: the wizard asks, the operator answers, the answer is discarded. The second is that the loader's compiled default and the value the template ships were pinned to each other by nothing. Flipping the default passed the whole suite; flipping the template only moved golden transcripts, which a regeneration absorbs silently. It matters because the install engine writes no BACKUP_FIREWALL_RULES at all when no front-end answered, so the effective value comes from the template on a fresh install and from the compiled default on a config that predates the key. If those disagree, one release means two things depending on how the operator got their config. Both sides of that comparison go through the real loader. A test-local parser was written first and thrown away: it was a second, weaker implementation of the thing under test, free to drift from the loader's own handling of quotes, comments, export prefixes and key aliases, and then agree with itself while disagreeing with production. A third test covers what the second cannot see. Pinning only the absent-key path leaves the loader free to read a MISSPELLED key -- verified: a typo at the read site passes the entire repo suite today -- so an explicit value is now asserted to be honoured. Verified by mutation. Correcting my own first count, which was inflated: of the five edits I originally listed, two were already caught at HEAD by existing tests. The coverage that is genuinely new is the stored-beats-answered guard, the loader default in both drift directions, and the misspelled-key read. Test-only: no production line changed, no golden moved. --- internal/config/config_test.go | 68 +++++++++++++++++++++++++ internal/installer/install_data_test.go | 52 +++++++++++++++++++ 2 files changed, 120 insertions(+) diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 5da7da44..07e96d09 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -1626,3 +1626,71 @@ func TestParseEnvFileHandlesExportLines(t *testing.T) { t.Fatalf("LOG_PATH = %q; want %q", got, "/logs") } } + +// loadEnvForTest writes content to a temp backup.env and loads it through the REAL +// loader, so both sides of a comparison are read by the code that ships. +func loadEnvForTest(t *testing.T, name, content string) *Config { + t.Helper() + path := filepath.Join(t.TempDir(), name) + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatalf("write %s: %v", name, err) + } + cfg, err := LoadConfigWithBaseDir(path, "/custom/base") + if err != nil { + t.Fatalf("LoadConfigWithBaseDir(%s): %v", name, err) + } + return cfg +} + +// TestBackupFirewallRulesDefaultMatchesShippedTemplate ties the loader's default to the +// value the shipped template carries, in BOTH directions. +// +// They agree today, and that agreement is load-bearing: the install engine writes no +// BACKUP_FIREWALL_RULES at all when no front-end answered, so what an operator ends up +// with is decided by the template on a fresh install and by this default on a config +// that predates the key. If the two ever disagree, the same release means two things +// depending on how the operator got their config -- and nothing else in the repo would +// say so. Flipping the loader default passes the entire suite; flipping the template +// only moves golden transcripts, which a regeneration absorbs silently. +// +// Both sides go through the real loader rather than a parser written for the test. +// A test-local parser would be a second, weaker implementation of the thing under test: +// it could drift from the loader's own handling of quotes, inline comments, `export` +// prefixes and key aliases, and then agree with itself while disagreeing with production. +// +// Unlike TestOptimizationAndSecurityDefaultsMatchTemplate above, this reads the shipped +// template instead of restating its values, so it keeps holding across a deliberate +// template change instead of having to be edited alongside one. (Verified: flipping a +// value in the shipped template leaves that neighbouring test passing.) +func TestBackupFirewallRulesDefaultMatchesShippedTemplate(t *testing.T) { + shipped := loadEnvForTest(t, "shipped.env", DefaultEnvTemplate()) + absent := loadEnvForTest(t, "absent.env", "# a config that predates the firewall toggle\n") + + if shipped.BackupFirewallRules != absent.BackupFirewallRules { + t.Fatalf("the shipped template yields BackupFirewallRules=%v but a config without the key yields %v; "+ + "the two must agree or a fresh install and an upgraded config disagree about firewall collection", + shipped.BackupFirewallRules, absent.BackupFirewallRules) + } +} + +// TestBackupFirewallRulesHonoursAnExplicitValue covers the other half, which the default +// test cannot see because it only ever exercises the absent-key path: that a value the +// operator actually wrote is READ. Without it, the loader could be looking up a +// misspelled key and every assertion about the default would still pass -- verified: a +// typo in the key name at config.go's read site passes the entire repo suite today. +func TestBackupFirewallRulesHonoursAnExplicitValue(t *testing.T) { + for _, tc := range []struct { + name string + content string + want bool + }{ + {name: "explicit true", content: "BACKUP_FIREWALL_RULES=true\n", want: true}, + {name: "explicit false", content: "BACKUP_FIREWALL_RULES=false\n", want: false}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := loadEnvForTest(t, "firewall.env", tc.content).BackupFirewallRules; got != tc.want { + t.Fatalf("BackupFirewallRules = %v, want %v", got, tc.want) + } + }) + } +} diff --git a/internal/installer/install_data_test.go b/internal/installer/install_data_test.go index 343e70a2..4f671a80 100644 --- a/internal/installer/install_data_test.go +++ b/internal/installer/install_data_test.go @@ -476,3 +476,55 @@ func TestApplyInstallDataEmailFallbackSendmail(t *testing.T) { } }) } + +// TestApplyInstallDataFirewallAnswerBeatsTheStoredValue pins the direction no golden +// covers: an operator CHANGING the firewall answer on an edit. +// +// The three characterization goldens all agree with whatever was already stored -- +// the two fresh ones have nothing stored, and edit_noop answers true over a stored +// true -- so nothing in the repo notices if the engine starts preferring the stored +// value over the answered one. That is a one-line change away (guarding the write on +// the key being absent) and it lands on the arm production actually takes: both +// wizards ASK this question, so the explicit arm is the live one. +// +// The failure it prevents is silent and total: the wizard asks, the operator answers, +// and the answer is discarded. +func TestApplyInstallDataFirewallAnswerBeatsTheStoredValue(t *testing.T) { + boolPtr := func(v bool) *bool { return &v } + answered := func(v bool) *InstallWizardData { + return &InstallWizardData{BaseDir: "/data", NotificationMode: "none", BackupFirewallRules: boolPtr(v)} + } + stored := func(v string) string { + return "TELEGRAM_ENABLED=false\nBACKUP_FIREWALL_RULES=" + v + "\n" + } + assertKey := func(t *testing.T, result, want string) { + t.Helper() + got, ok := parseEnvTemplate(result)["BACKUP_FIREWALL_RULES"] + if !ok || got != want { + t.Fatalf("BACKUP_FIREWALL_RULES = %q (present=%v), want %q:\n%s", got, ok, want, result) + } + // A second line would let the loader read the intended value while the file + // carries two sources of truth, so the value assertion alone is not enough. + if n := strings.Count(result, "BACKUP_FIREWALL_RULES"); n != 1 { + t.Fatalf("the key must appear exactly once, found %d:\n%s", n, result) + } + } + + t.Run("an explicit no turns off a stored yes", func(t *testing.T) { + assertKey(t, mustApply(t, stored("true"), answered(false)), "false") + }) + + t.Run("an explicit yes turns on a stored no", func(t *testing.T) { + assertKey(t, mustApply(t, stored("false"), answered(true)), "true") + }) +} + +// mustApply runs ApplyInstallData and fails the test on error. +func mustApply(t *testing.T, base string, data *InstallWizardData) string { + t.Helper() + result, err := ApplyInstallData(base, data) + if err != nil { + t.Fatalf("ApplyInstallData: %v", err) + } + return result +} From 068acee2017a5d673f3984455a4bdf472cb7a162 Mon Sep 17 00:00:00 2001 From: tis24dev Date: Mon, 3 Aug 2026 20:09:39 +0200 Subject: [PATCH 24/50] fix(config): align the PXAR scan default with the shipped template, and pin every boolean pair The compiled default was true while the template shipped PXAR_SCAN_ENABLE=false, so one release behaved two ways. A fresh install had PBS datastore scanning OFF; a config old enough to predate the key had it ON and walked every datastore. Nobody chose the second behaviour -- it fell out of the mismatch -- and the expensive side is the wrong side for a value nobody set. The template is the curated statement of intent, so the compiled default now follows it. Behaviour change, stated plainly: an installation whose backup.env carries neither PXAR_SCAN_ENABLE nor BACKUP_PXAR_FILES stops scanning datastores after this. Any config written from the template is unaffected, because it carries the key. Nothing broke when the default changed, which is the defect restated: no test covered the absent-key path for this field, or for any other. So the second half generalises the pin instead of adding a second single-key test. TestBoolDefaultsMatchTheShippedTemplate loads the shipped template and a config with no keys at all -- both through the REAL loader, not a parser written for the test -- and asserts every exported boolean setting agrees across the two. It replaces the firewall-specific default test added a commit ago, which it subsumes. The generalised form is the point. This defect was found once, by hand, while looking at a different key; a single-key test would have pinned the one pair somebody happened to examine and left the rest unwatched. A field that must legitimately diverge now needs an explicit exception with its reason written next to it. There are none. Verified by mutation: restoring the PXAR defect, flipping the firewall default, flipping a template value, and flipping the default of an UNRELATED field (BACKUP_SYNC_JOBS) are all caught. The last one is what proves the test measures every boolean rather than the two that were being looked at. Not touched: internal/backup/collector.go's own BackupPxarFiles default, a separate layer that the orchestrator always overwrites from the loaded config. --- internal/config/config.go | 8 +++- internal/config/config_test.go | 70 ++++++++++++++++++++++------------ 2 files changed, 52 insertions(+), 26 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index facf6cac..420570cb 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -961,7 +961,13 @@ func (c *Config) parsePBSSettings() { networkFallback := c.getBoolWithFallback([]string{"BACKUP_NETWORK_CONFIGS", "BACKUP_NETWORK_CONFIG"}, true) c.BackupPBSNetworkConfig = c.getBool("BACKUP_PBS_NETWORK_CONFIG", networkFallback) c.BackupPruneSchedules = c.getBool("BACKUP_PRUNE_SCHEDULES", true) - c.BackupPxarFiles = c.getBoolWithFallback([]string{"PXAR_SCAN_ENABLE", "BACKUP_PXAR_FILES"}, true) + // Default false to match the shipped template. It used to compile to true while the + // template shipped PXAR_SCAN_ENABLE=false, which meant one release behaved two ways: + // a fresh install had datastore scanning OFF, while a config old enough to predate + // the key had it ON and walked every PBS datastore. Nobody chose the second + // behaviour -- it fell out of the mismatch -- and the expensive side is the wrong + // side for a value nobody set. + c.BackupPxarFiles = c.getBoolWithFallback([]string{"PXAR_SCAN_ENABLE", "BACKUP_PXAR_FILES"}, false) c.PxarDatastoreConcurrency = c.getInt("PXAR_SCAN_DS_CONCURRENCY", 3) c.PxarFileIncludePatterns = normalizeList(c.getStringSliceWithFallback([]string{"PXAR_FILE_INCLUDE_PATTERN", "PXAR_INCLUDE_PATTERN"}, nil)) c.PxarFileExcludePatterns = normalizeList(c.getStringSlice("PXAR_FILE_EXCLUDE_PATTERN", nil)) diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 07e96d09..38a36e11 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -11,6 +11,7 @@ import ( "math/big" "os" "path/filepath" + "reflect" "strings" "testing" "time" @@ -1642,34 +1643,53 @@ func loadEnvForTest(t *testing.T, name, content string) *Config { return cfg } -// TestBackupFirewallRulesDefaultMatchesShippedTemplate ties the loader's default to the -// value the shipped template carries, in BOTH directions. +// TestBoolDefaultsMatchTheShippedTemplate asserts, for EVERY boolean setting, that the +// compiled default agrees with the value the shipped template carries. // -// They agree today, and that agreement is load-bearing: the install engine writes no -// BACKUP_FIREWALL_RULES at all when no front-end answered, so what an operator ends up -// with is decided by the template on a fresh install and by this default on a config -// that predates the key. If the two ever disagree, the same release means two things -// depending on how the operator got their config -- and nothing else in the repo would -// say so. Flipping the loader default passes the entire suite; flipping the template -// only moves golden transcripts, which a regeneration absorbs silently. +// The two are different code paths that reach the same operator. The install engine does +// not write every key, and a config can predate any key, so the effective value comes +// from the template on a fresh install and from the compiled default on a config that +// lacks the line. When those disagree, one release behaves two ways depending on how the +// operator got their config -- and nothing else in the repo says so: flipping a compiled +// default passes the entire suite, and flipping a template value only moves golden +// transcripts, which a regeneration absorbs silently. // -// Both sides go through the real loader rather than a parser written for the test. -// A test-local parser would be a second, weaker implementation of the thing under test: -// it could drift from the loader's own handling of quotes, inline comments, `export` -// prefixes and key aliases, and then agree with itself while disagreeing with production. +// This is the generalised form on purpose. The same defect was found once by hand +// (PXAR_SCAN_ENABLE shipped false while the code compiled true, so a config old enough +// to predate the key walked every PBS datastore) and a single-key test would have pinned +// the one pair somebody happened to look at while the rest stayed unwatched. // -// Unlike TestOptimizationAndSecurityDefaultsMatchTemplate above, this reads the shipped -// template instead of restating its values, so it keeps holding across a deliberate -// template change instead of having to be edited alongside one. (Verified: flipping a -// value in the shipped template leaves that neighbouring test passing.) -func TestBackupFirewallRulesDefaultMatchesShippedTemplate(t *testing.T) { - shipped := loadEnvForTest(t, "shipped.env", DefaultEnvTemplate()) - absent := loadEnvForTest(t, "absent.env", "# a config that predates the firewall toggle\n") - - if shipped.BackupFirewallRules != absent.BackupFirewallRules { - t.Fatalf("the shipped template yields BackupFirewallRules=%v but a config without the key yields %v; "+ - "the two must agree or a fresh install and an upgraded config disagree about firewall collection", - shipped.BackupFirewallRules, absent.BackupFirewallRules) +// Both sides go through the real loader rather than a parser written for the test: a +// test-local parser is a second, weaker implementation of the thing under test, free to +// drift from the loader's own handling of quotes, comments, export prefixes and key +// aliases and then agree with itself while disagreeing with production. +// +// A field that must legitimately diverge belongs in an explicit exception list here, with +// the reason next to it. There are none today. +func TestBoolDefaultsMatchTheShippedTemplate(t *testing.T) { + shipped := reflect.ValueOf(*loadEnvForTest(t, "shipped.env", DefaultEnvTemplate())) + absent := reflect.ValueOf(*loadEnvForTest(t, "absent.env", "# a config that carries no keys at all\n")) + + var diverging []string + typ := shipped.Type() + checked := 0 + for i := 0; i < typ.NumField(); i++ { + field := typ.Field(i) + if !field.IsExported() || field.Type.Kind() != reflect.Bool { + continue + } + checked++ + if got, want := absent.Field(i).Bool(), shipped.Field(i).Bool(); got != want { + diverging = append(diverging, fmt.Sprintf("%s: shipped template says %v, compiled default says %v", field.Name, want, got)) + } + } + if checked == 0 { + t.Fatal("no exported bool settings found; this test has stopped measuring anything") + } + if len(diverging) > 0 { + t.Fatalf("%d of %d boolean settings disagree between the shipped template and the compiled default.\n"+ + "Each means a fresh install and a config that predates the key behave differently:\n %s", + len(diverging), checked, strings.Join(diverging, "\n ")) } } From 2a7a844c13885e018e86f6da264a2585d2d62d3e Mon Sep 17 00:00:00 2001 From: tis24dev Date: Mon, 3 Aug 2026 21:21:55 +0200 Subject: [PATCH 25/50] Drop the agent-tool name from two comments Neither comment carried information that needed the vendor name: the .gitignore header describes a local-only workspace, and the trackedFiles doc comment enumerates excluded trees where "the local agent-tooling directories" is equally precise. The functional occurrences are deliberately untouched. The two .gitignore patterns are what keep those paths out of the repo, and the skip-list key in trackedFiles must match the directory name literally for the fallback walk to skip it. Comment-only: no statement or expression changed. --- .gitignore | 2 +- internal/sourceguard/sourceguard_test.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 662b9aa9..e0a99d58 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,7 @@ /AGENTS.md /CLAUDE.md -# GSD workspace — all Claude/GSD planning projects, local-only +# GSD workspace — local-only planning projects, never version-controlled /gsd/ # Build artifacts — compiled binary (root build) and Makefile output dir diff --git a/internal/sourceguard/sourceguard_test.go b/internal/sourceguard/sourceguard_test.go index 743c8507..3d377239 100644 --- a/internal/sourceguard/sourceguard_test.go +++ b/internal/sourceguard/sourceguard_test.go @@ -156,9 +156,9 @@ func repoRoot(t *testing.T) string { // trackedFiles returns repo-relative paths of git-tracked files. git ls-files // yields only tracked files and already excludes gitignored and vendored trees -// (vendor/, diagnostics/, .claude/, .superpowers/). If git cannot run in this -// sandbox it falls back to a filepath.Walk that skips those trees, so the guard -// never silently degrades to scanning nothing. +// (vendor/, diagnostics/, and the local agent-tooling directories). If git +// cannot run in this sandbox it falls back to a filepath.Walk that skips those +// trees, so the guard never silently degrades to scanning nothing. func trackedFiles(t *testing.T, root string) []string { t.Helper() if out, err := exec.Command("git", "-C", root, "ls-files", "-z").Output(); err == nil { From d1039af2207cecfe2889a98f224936d2c333be44 Mon Sep 17 00:00:00 2001 From: tis24dev Date: Mon, 3 Aug 2026 22:15:33 +0200 Subject: [PATCH 26/50] fix(restore): keep the cluster database out of the analysis-failure fallback The full-restore fallback leaves NeedsClusterRestore off, so pve-cluster keeps running and /etc/pve stays mounted, but the extraction still wrote /var/lib/pve-cluster/. That overwrites config.db under a live pmxcfs holding it open as SQLite, and corosync carries the damage to the rest of the cluster. The /etc/pve guard in restore_archive_entries.go does not cover this path: the database lives elsewhere. skipPath now drops the cluster category's own paths, read from categories.go, and the comment on NeedsClusterRestore records that the two decisions are load-bearing for each other. The prefix match that isExportOnlyPath carried is now matchesAnyArchivePrefix, shared by both arms. The skip is unconditional; only the operator-facing warning is gated on a PVE host. --- .../orchestrator/restore_workflow_ui_full.go | 71 +++++++++++---- .../restore_workflow_ui_full_safety.go | 22 +++-- .../restore_workflow_ui_full_safety_test.go | 87 +++++++++++++++++++ 3 files changed, 160 insertions(+), 20 deletions(-) diff --git a/internal/orchestrator/restore_workflow_ui_full.go b/internal/orchestrator/restore_workflow_ui_full.go index 9b1e0b71..b6404b61 100644 --- a/internal/orchestrator/restore_workflow_ui_full.go +++ b/internal/orchestrator/restore_workflow_ui_full.go @@ -46,6 +46,13 @@ func (f *fullRestoreUIFlow) extract() error { if f.safeFstabMerge() { f.logger.Warning("Full restore safety: /etc/fstab will not be overwritten; Smart Merge will be applied after extraction.") } + // Announced only on a PVE host: skipPath drops these entries whatever the detected + // system type, but telling a PBS operator about a cluster database is noise. + if f.plan != nil && f.plan.SystemType.SupportsPVE() { + if paths := clusterDBArchivePaths(); len(paths) > 0 { + f.logger.Warning("Full restore safety: %s will NOT be restored. This fallback never stops pve-cluster, so writing the cluster database under a live pmxcfs would corrupt it. Restore that category with a selective restore once the archive can be analyzed.", strings.Join(paths, ", ")) + } + } if err := extractPlainArchive(f.ctx, f.prepared.ArchivePath, f.destRoot, f.logger, f.skipPath); err != nil { return err } @@ -56,35 +63,69 @@ func (f *fullRestoreUIFlow) extract() error { return nil } -// skipPath keeps two classes of entry out of a plain extraction: /etc/fstab, which -// is merged afterwards instead of overwritten, and everything belonging to an -// ExportOnly category. The selective path never writes export-only content to system -// paths (splitRestoreCategories routes it to an export directory); before this, the +// skipPath keeps three classes of entry out of a plain extraction: /etc/fstab, which +// is merged afterwards instead of overwritten; the PVE cluster database, which this +// fallback has no safe way to write; and everything belonging to an ExportOnly +// category. The selective path never writes export-only content to system paths +// (splitRestoreCategories routes it to an export directory); before this, the // fallback wrote /etc/proxmox-backup/ and /var/lib/proxsave-info/ straight to /. // -// The prefixes come from the plan's own ExportCategories, so there is no second list -// to keep in step with categories.go. +// The prefixes come from the plan's own ExportCategories and from categories.go, so +// there is no second list to keep in step with them. func (f *fullRestoreUIFlow) skipPath(name string) bool { clean := normalizeArchiveEntryPath(name) if f.safeFstabMerge() && clean == "etc/fstab" { return true } + if matchesAnyArchivePrefix(clean, clusterDBArchivePaths()) { + return true + } return f.isExportOnlyPath(clean) } +// clusterDBArchivePaths returns the archive prefixes holding the PVE cluster +// database, read from categories.go rather than restated here. +// +// The fallback must never write them. runFullRestore leaves NeedsClusterRestore off, +// so pve-cluster keeps running and /etc/pve stays mounted; extracting config.db under +// a live pmxcfs that holds it open as SQLite corrupts it, and on a clustered node +// corosync would carry the damage to every other member. The /etc/pve block in +// restore_archive_entries.go does not cover this: the database lives under +// /var/lib/pve-cluster/, which no other guard touches. +func clusterDBArchivePaths() []string { + cat := GetCategoryByID("pve_cluster", GetAllCategories()) + if cat == nil { + return nil + } + return cat.Paths +} + func (f *fullRestoreUIFlow) isExportOnlyPath(clean string) bool { - if f.plan == nil || clean == "" { + if f.plan == nil { return false } for _, cat := range f.plan.ExportCategories { - for _, p := range cat.Paths { - prefix := normalizeArchiveEntryPath(p) - if prefix == "" { - continue - } - if clean == prefix || strings.HasPrefix(clean, strings.TrimSuffix(prefix, "/")+"/") { - return true - } + if matchesAnyArchivePrefix(clean, cat.Paths) { + return true + } + } + return false +} + +// matchesAnyArchivePrefix reports whether clean sits at or under any of the given +// category paths, normalizing both sides so a "./var/lib/x/" category path and a +// "var/lib/x/y" archive entry compare correctly. +func matchesAnyArchivePrefix(clean string, paths []string) bool { + if clean == "" { + return false + } + for _, p := range paths { + prefix := normalizeArchiveEntryPath(p) + if prefix == "" { + continue + } + if clean == prefix || strings.HasPrefix(clean, strings.TrimSuffix(prefix, "/")+"/") { + return true } } return false diff --git a/internal/orchestrator/restore_workflow_ui_full_safety.go b/internal/orchestrator/restore_workflow_ui_full_safety.go index fc1b1623..942b40a8 100644 --- a/internal/orchestrator/restore_workflow_ui_full_safety.go +++ b/internal/orchestrator/restore_workflow_ui_full_safety.go @@ -81,11 +81,23 @@ func (w *restoreUIWorkflowRun) synthesizeFullRestorePlan() *RestorePlan { categories := categoriesPresentUnderRoot(GetAllCategories(), w.destRoot) plan := PlanRestore(false, categories, w.systemType, RestoreModeFull) - // A plain extraction can never write /etc/pve: restore_archive_entries.go blocks - // those entries unconditionally. Stopping the cluster and unmounting /etc/pve - // would therefore be pure disruption, so the cluster path stays off even though a - // full category set nominally selects it. PBS services are left to PlanRestore: - // /etc/proxmox-backup IS written by this extraction. + // The cluster stays up: this fallback runs because the archive could not be + // analyzed, so it cannot know whether the archive even holds usable cluster data, + // and stopping pve-cluster plus unmounting /etc/pve on that guess is disruption + // bought with nothing. + // + // That decision is only safe because the extraction skips the cluster database + // too - see clusterDBArchivePaths in restore_workflow_ui_full.go. The two must + // stay together: leaving this false while letting /var/lib/pve-cluster/ through + // would write config.db under a live pmxcfs. The /etc/pve block in + // restore_archive_entries.go is NOT that guarantee; it covers a different path. + // + // NeedsPBSServices is left as PlanRestore computed it, but note what that buys: + // pbs_config is ExportOnly over the whole ./etc/proxmox-backup/ prefix, and + // skipPath filters by path rather than by category, so every entry under it is + // dropped no matter which category owns it. This extraction therefore writes no + // PBS configuration at all, yet still stops the PBS services. Left alone on + // purpose - stopping them is harmless, and narrowing it is a separate decision. plan.NeedsClusterRestore = false return plan } diff --git a/internal/orchestrator/restore_workflow_ui_full_safety_test.go b/internal/orchestrator/restore_workflow_ui_full_safety_test.go index fae4e507..34b034bf 100644 --- a/internal/orchestrator/restore_workflow_ui_full_safety_test.go +++ b/internal/orchestrator/restore_workflow_ui_full_safety_test.go @@ -120,3 +120,90 @@ func TestFullRestoreFallbackKeepsSafetyInvariants(t *testing.T) { t.Fatalf("fallback ran without a safety backup; entries=%v", entries) } } + +// TestFullRestoreFallbackNeverWritesClusterDB pins the one thing the fallback must +// not do on a PVE node. It leaves NeedsClusterRestore off, so pve-cluster keeps +// running and /etc/pve stays mounted; if the extraction let /var/lib/pve-cluster/ +// through, config.db would be overwritten under a live pmxcfs holding it open as +// SQLite, and corosync would carry the damage to the rest of the cluster. +// +// The /etc/pve guard in restore_archive_entries.go does NOT cover this: the database +// lives elsewhere. Deleting the cluster arm of skipPath turns this test RED. +func TestFullRestoreFallbackNeverWritesClusterDB(t *testing.T) { + origRestoreFS := restoreFS + origRestoreCmd := restoreCmd + origRestoreSystem := restoreSystem + origCompatFS := compatFS + origPrepare := prepareRestoreBundleFunc + origAnalyze := analyzeRestoreArchiveFunc + origSafetyFS := safetyFS + t.Cleanup(func() { + restoreFS = origRestoreFS + restoreCmd = origRestoreCmd + restoreSystem = origRestoreSystem + compatFS = origCompatFS + prepareRestoreBundleFunc = origPrepare + analyzeRestoreArchiveFunc = origAnalyze + safetyFS = origSafetyFS + }) + + fakeFS := NewFakeFS() + t.Cleanup(func() { _ = os.RemoveAll(fakeFS.Root) }) + restoreFS = fakeFS + compatFS = fakeFS + safetyFS = fakeFS + restoreCmd = runOnlyRunner{} + restoreSystem = fakeSystemDetector{systemType: SystemTypePVE} + + // The live cluster database pmxcfs is holding open. It must still hold this + // content after the fallback runs. + if err := fakeFS.AddFile("/var/lib/pve-cluster/config.db", []byte("live-cluster-db\n")); err != nil { + t.Fatalf("fakeFS.AddFile: %v", err) + } + + tmpTar := filepath.Join(t.TempDir(), "bundle.tar") + if err := writeTarFile(tmpTar, map[string]string{ + "etc/hosts": "127.0.0.1 localhost\n", + "var/lib/pve-cluster/config.db": "from-archive\n", + }); err != nil { + t.Fatalf("writeTarFile: %v", err) + } + tarBytes, err := os.ReadFile(tmpTar) + if err != nil { + t.Fatalf("ReadFile tar: %v", err) + } + if err := fakeFS.WriteFile("/bundle.tar", tarBytes, 0o640); err != nil { + t.Fatalf("fakeFS.WriteFile: %v", err) + } + + prepareRestoreBundleFunc = stubPreparedRestoreBundle("/bundle.tar", &backup.Manifest{ + CreatedAt: time.Unix(1700000000, 0), + ClusterMode: "cluster", + ProxmoxType: "pve", + ScriptVersion: "vtest", + }) + analyzeRestoreArchiveFunc = func(archivePath string, logger *logging.Logger) ([]Category, *RestoreDecisionInfo, error) { + return nil, nil, errors.New("boom") + } + + logger := logging.New(types.LogLevelError, false) + cfg := &config.Config{BaseDir: "/base"} + ui := &fakeRestoreWorkflowUI{confirmRestore: true, confirmCompatible: true, continuePBSServices: true} + + if err := runRestoreWorkflowWithUI(context.Background(), cfg, logger, "vtest", ui); err != nil { + t.Fatalf("runRestoreWorkflowWithUI error: %v", err) + } + + // Still a FULL restore: everything outside the cluster database is written. + if _, err := fakeFS.ReadFile("/etc/hosts"); err != nil { + t.Fatalf("fallback must still extract /etc/hosts: %v", err) + } + + live, err := fakeFS.ReadFile("/var/lib/pve-cluster/config.db") + if err != nil { + t.Fatalf("read live cluster database: %v", err) + } + if strings.TrimSpace(string(live)) != "live-cluster-db" { + t.Fatalf("fallback overwrote the live cluster database under a running pmxcfs: %q", string(live)) + } +} From aea1b4c513b7813dd527ffd752c14c623d357320 Mon Sep 17 00:00:00 2001 From: tis24dev Date: Tue, 4 Aug 2026 04:12:06 +0200 Subject: [PATCH 27/50] fix(daemon): survive a backup child that can never be reaped A child parked in TASK_UNINTERRUPTIBLE behind a dead NFS/CIFS mount never dequeues the SIGKILL os/exec sends after WaitDelay, so it is never reaped and wait4 never returns. Cmd.Wait blocks in Process.Wait before it ever reads the watchdog result, so cmd.Run() never returned either: runOnce is called synchronously from scheduleLoop, so the scheduler wedged and no further backup was ever scheduled, SIGTERM could not shut the daemon down, and the hang report sitting downstream of cmd.Run() was unreachable for exactly the case it documented. Meanwhile the heartbeat goroutine kept reporting the host green. superviseChild now waits with a deadline. A child that dies late but dies is reaped and reported exactly as before; one still unreaped after SIGTERM, SIGKILL and a further reap slack is abandoned. The daemon then drives the backup check DOWN, records the abandon, degrades the service-alive check and exits so systemd restarts it clean rather than leaking the goroutine and descriptors stranded behind a child that cannot be collected. The alive degrade has to outlive the process or the restarted daemon's immediate first beat would flip the check green ten seconds later and fire a "recovered" alert over a host where no backup can run, so it is persisted as a marker in the identity dir. Lifting it needs evidence, not an exit code: the backup lock is checked LAST, after the directory and disk-space gates that the same dead mount fails first, so a run can reach a real exit code without ever having reached the orphan's lock. The marker therefore carries (pid, starttime) -- the pid alone is not an identifier, since the kernel recycles the number within a boot -- and the degrade lifts only once that pair is provably gone, or a run proves it reached the lock when the marker names no usable pid. The systemd unit is unchanged. KillMode is deliberately left at control-group: the cgroup-wide kill is the only thing that collects the abandoned child's own tar/pigz/rclone descendants, which nothing else signals. --- cmd/proxsave/daemon.go | 1328 ++++++++++++++++++++- cmd/proxsave/daemon_abandon_test.go | 1709 +++++++++++++++++++++++++++ cmd/proxsave/daemon_service.go | 17 + cmd/proxsave/daemon_service_test.go | 17 + cmd/proxsave/daemon_test.go | 14 +- internal/health/abandon.go | 100 ++ internal/health/abandon_test.go | 64 + internal/health/reporter.go | 22 +- internal/health/reporter_test.go | 25 + internal/health/status.go | 9 +- 10 files changed, 3265 insertions(+), 40 deletions(-) create mode 100644 cmd/proxsave/daemon_abandon_test.go create mode 100644 internal/health/abandon.go create mode 100644 internal/health/abandon_test.go diff --git a/cmd/proxsave/daemon.go b/cmd/proxsave/daemon.go index 38c1197c..a6a0b01d 100644 --- a/cmd/proxsave/daemon.go +++ b/cmd/proxsave/daemon.go @@ -12,8 +12,10 @@ import ( "os/exec" "os/signal" "sort" + "strconv" "strings" "sync" + "sync/atomic" "syscall" "time" @@ -30,6 +32,53 @@ import ( const ( // daemonKillGrace is how long a hung child gets between SIGTERM and SIGKILL. daemonKillGrace = 30 * time.Second + // daemonReapSlack is how much longer, AFTER that SIGKILL, the daemon still waits for the + // child to be REAPED before declaring it unreapable and abandoning it. A process SIGKILL + // can actually kill is reaped within microseconds of the signal, so this margin is only + // ever spent on a child that is already lost: a task parked in TASK_UNINTERRUPTIBLE (D + // state -- a dead NFS/CIFS mount, a wedged block device) never dequeues the signal and + // never will be reaped. It is deliberately generous enough that a merely slow teardown on + // a loaded host is still reaped normally rather than misjudged, and deliberately short + // enough that abandoning during a shutdown (killGrace + this = 45s) stays inside a stock + // host's DefaultTimeoutStopSec of 90s. The unit does NOT pin that value (buildDaemonUnit + // leaves the kill directives at their defaults on purpose), so a host that lowered it + // below 45s simply has systemd SIGKILL the daemon mid-teardown -- which is exactly what + // happened on every host before this path existed. See superviseChild. + daemonReapSlack = 15 * time.Second + // daemonAliveInterlockWait caps how long the abandon path waits for aliveMu before it + // reports anyway. An in-flight beat holds that lock for one ping (pingTimeout, 10s) plus a + // small status-file write, so this is comfortably longer than any legitimate hold. It + // exists because the ordering the lock buys -- the degrade /fail transmitting last -- is a + // nicety, while runOnce returning is the invariant: no lock holder, however it got stuck, + // may be able to wedge the scheduler again. + daemonAliveInterlockWait = 15 * time.Second + // daemonAbandonIOWait bounds the LOCAL filesystem writes the abandon path performs: the + // abandoned-child marker, and the status-file record of the hang ping. Everything else on + // that path is a network ping the Reporter already caps at pingTimeout. It exists because + // the abandon path runs, by definition, on a host with wedged I/O -- a dead NFS/CIFS mount, + // a hung block device. If BaseDir sits on that same filesystem, an unbounded + // os.MkdirAll/os.WriteFile/os.Rename blocks exactly like the wait4 that started all this, + // and the wedge is straight back one layer out, after having already cost the two + // monitoring signals it was ordered ahead of. Generous next to a healthy write + // (microseconds) and short next to the pings around it. + daemonAbandonIOWait = 5 * time.Second + // daemonProcProbeWait bounds the /proc read that identifies an abandoned child (see + // abandonedChildGone). /proc is a pseudo-filesystem and never lives on the dead mount, but + // the file it serves is rendered from the very task that is parked in the kernel, and + // /proc//cmdline in particular is read under that task's mmap lock -- which a page + // fault on the wedged mount can be holding. That probe runs on the SCHEDULER's goroutine + // (clearAbandonMarkerOnCompletedRun) as well as on the beat, so an unbounded one would put + // an unbounded wait back on the path this whole change exists to bound. Expiry is not an + // error: it means the task is demonstrably still in the kernel, which is the same answer + // "still there" the probe would have given. + daemonProcProbeWait = 2 * time.Second + // daemonLoopDrainWait caps the join on the background loops when the daemon is unwinding + // after an abandon. They return immediately on their context being cancelled, EXCEPT a + // beat already inside buildReporter, which can be parked on the cross-process relay-secret + // flock (identity.LockNotifySecret takes LOCK_EX with no deadline and no context). The + // abandon pings have already been sent by then and the process is about to die, so an + // undrainable loop must not be able to hold the exit -- and with it the systemd restart. + daemonLoopDrainWait = 15 * time.Second // logTailBytes bounds the log excerpt POSTed with a non-success outcome. logTailBytes = 8 * 1024 // defaultMaxRunDuration is the watchdog fallback when MAX_RUN_DURATION is unset. A @@ -52,6 +101,9 @@ const ( // implements it. An interface so the scheduler/watchdog is testable with a fake. type backupReporter interface { Heartbeat(ctx context.Context) error + // AliveDegraded drives the service-alive check DOWN. Sent once, from the abandon + // path, so the monitor never shows a green service behind dead backups. + AliveDegraded(ctx context.Context, reason string) error RunStarted(ctx context.Context, rid string) error RunFinished(ctx context.Context, rid string, exitCode int, logTail string) error RunHang(ctx context.Context, rid string, timeout time.Duration, logTail string) error @@ -84,8 +136,152 @@ type daemon struct { fetchWarned bool // centralized fetch already warned once (throttle recurring WARN) updateWarned bool // an update is already known available (WARN once per transition) provisionRetryAt time.Time // next relay-secret self-heal attempt; guarded by mu + // aliveMu orders the TRANSMISSIONS to the service-alive check: one beat's ping+record, or + // the one abandon degrade, never both at once. A bare latch cannot do this. beat() reads + // it, then spends up to a full pingTimeout inside r.Heartbeat, so a beat that entered + // before the latch closed would still deliver its SUCCESS ping AFTER the degrade's /fail + // and re-green the check remotely -- exactly the failure the latch was written to prevent. + // The mutex makes the abandon wait for that in-flight beat instead, so the /fail is the + // last word on the check. + // + // What it must NEVER cover is URL RESOLUTION. buildReporter can reach a cross-process + // flock (fetchCentralized -> maybeProvisionRelaySecret -> identity.LockNotifySecret -> + // syscall.Flock LOCK_EX, which honours neither a deadline nor the context), and an + // unbounded hold here would put an unbounded wait right back on the abandon path -- the + // same wedge this whole change exists to remove, one layer down. So beat resolves the + // reporter BEFORE taking the lock, and the only things under it are bounded: one ping + // (the reporter caps it at pingTimeout) and one status-file write. abandonChild's own + // acquisition is additionally deadline-capped (see lockAliveWithin), so no lock holder can + // ever stop runOnce from returning. + // + // Lock order: aliveMu is outermost (code under it takes statusMu); nothing takes it while + // holding mu or statusMu. + aliveMu sync.Mutex + // aliveSilenced latches when the daemon has driven the service-alive check DOWN on its way + // OUT (abandonChild). Every later beat is then dropped WHOLE -- no ping, so nothing + // re-greens the check between that /fail and the exit, and no local record, so a daemon + // that is at that moment dying does not refresh its own liveness timestamp. Atomic rather + // than aliveMu-guarded so the abandon can close it even if the interlock deadline expires. + aliveSilenced atomic.Bool + // aliveDegraded mirrors the on-disk abandon marker (health.AbandonRecord) read at startup: + // a PREVIOUS process abandoned an unreapable child, and the orphan is still holding the + // backup lock. This daemon is alive and must keep saying so LOCALLY (it records every beat + // as usual, so the run-side panel and health.Diagnose stay honest), but each beat pings + // /fail instead of success, so the remote alive check stays DOWN for as long as backups + // are dead. Without it the restarted daemon's immediate first beat would flip the check + // back UP about ten seconds after the /fail, and fire a "recovered" alert over a host on + // which no backup can run. + // + // It is cleared -- marker and all -- the moment anything proves backups are no longer dead: + // a supervised run that reaches a real exit code (runOnce), a standalone run that hands one + // off (processManualOutcome), or the orphan simply being gone from the host + // (reviewAbandonDegrade, checked at startup and on every beat). It is never SET at all when + // backups are administratively off, because then nothing could ever clear it. A degrade that + // cannot be lifted is not a safety net, it is a false RED on the check that pages people -- + // the exact mirror of the false green it exists to remove. + aliveDegraded atomic.Bool + // abandonMarkerOnDisk records that a marker file may be sitting in the identity dir -- this + // process either read one at startup or could not tell (an unreadable marker is still a + // marker). It is what makes the removal in clearAbandonMarker unconditional on the degrade: + // a process that hit ReadAbandon's error branch never degrades, and must still delete the + // file once a run proves the wedge is over, or it lingers to resurrect the degrade the + // moment it becomes readable again. False on the overwhelmingly common path, so a normal + // run pays one atomic load. + abandonMarkerOnDisk atomic.Bool + // abandonNote explains that degrade in the ping body (the orphan's pid and run id). Set in + // run() before any loop starts, and REPLACED once a supervised or standalone run has + // completed over an orphan that is still on this host -- a state the degrade deliberately + // survives (clearAbandonMarkerOnCompletedRun), and one in which the original wording ("no + // backup has completed since") is a false statement transmitted on every beat for as long + // as the degrade stands. The whole point of the degrade is that the monitor is told the + // truth; a body that ages into a lie is the same defect in miniature. + // + // An atomic pointer because the replacement happens on the scheduler (runOnce) or the + // SIGUSR1 waker (processManualOutcome) while beat reads it on the heartbeat goroutine. It + // takes no lock, so it cannot interact with aliveMu/abandonMu ordering. nil means no + // degrade was ever raised and nothing reads it. + abandonNote atomic.Pointer[string] + // abandonPID / abandonStart identify the orphan named by that marker: its pid and its + // start time in clock ticks (0 when the marker carried neither, or carried a pid this + // process is not going to act on). Both are set ONLY on the branch that actually raises + // the degrade, and are read-only afterwards; they are what lets the degrade be + // re-validated against the host instead of being believed forever. A marker this process + // retires at startup leaves them at 0 on purpose: after a reboot that number belongs to + // something else entirely, and a daemon that kept probing it would report, after every + // successful backup, that it is holding DOWN a check that is in fact green. + abandonPID int + abandonStart uint64 + // abandonMu serializes the two operations that decide what the SUCCESSOR daemon inherits: + // writing the marker for a child this process just abandoned, and removing an inherited + // one because the wedge is over. Without it the two race on the beat/scheduler boundary -- + // a clear that has already passed its guard, and is parked in the unlink, lands on the + // marker abandonChild wrote moments later, and the next daemon comes up green over a live + // orphan. Both sides are deadline-bounded underneath it (runWithin), so the hold is + // bounded too and the abandon path can never be stalled on it. + // + // Lock order: innermost. Nothing is taken while it is held. + abandonMu sync.Mutex + // abandonMarkerWritten latches once THIS process has persisted a marker for a child of its + // OWN. From that instant the file in the identity dir describes the orphan the successor + // must inherit -- not the inherited one any clear path in this process is reasoning about + // -- so no clear may remove it. A latch rather than a comparison because there is nothing + // to compare: both markers live at the same path. + abandonMarkerWritten atomic.Bool + // abandonRec is the record behind that latch, published by persistAbandonMarker. It exists + // for one reader: a marker REMOVAL that stayed parked in the kernel past its deadline, was + // therefore no longer covered by abandonMu when it finally landed, and may have deleted the + // successor's inheritance on its way out. That straggler puts this record back. An atomic + // pointer rather than a mutex-guarded field so the straggler -- which runs on a goroutine + // its own caller spawned while HOLDING abandonMu -- can never deadlock against it. + abandonRec atomic.Pointer[health.AbandonRecord] + // probeInFlight latches while an orphan-identity /proc read is still parked in the kernel + // (see pidIsAbandonedChild). The read has a deadline but no cancellation -- nothing in + // userspace can cancel a syscall the kernel is holding -- so a probe that expires leaves its + // goroutine, and the open /proc file descriptor inside it, behind. reviewAbandonDegrade runs + // that probe once per HEARTBEAT for the entire life of a degraded daemon, so without this + // latch a host whose orphan blocks the read accumulates one stranded goroutine and one + // stranded fd every beat, forever, on the one host an operator is actively investigating. + // With it at most ONE read is ever outstanding: while it is set the answer is taken from the + // latch itself ("still there", the same answer an expiry gives), and probing resumes by + // itself the moment the parked read finally returns. + probeInFlight atomic.Bool // newBackupCmd builds the child backup command; overridable in tests. newBackupCmd func(ctx context.Context) *exec.Cmd + // killGraceOverride / reapWaitOverride replace daemonKillGrace and the reap deadline in + // tests; zero means the production values. Fields rather than config knobs: a shorter + // grace is never the right answer in production, but a test cannot spend a 45s wall clock + // proving the abandon path exists. Set at construction and never written afterwards. + killGraceOverride time.Duration + reapWaitOverride time.Duration + // pidAliveOverride replaces the WHOLE orphan probe -- kill(2) liveness and the /proc + // identity check behind it -- for an abandoned child's pid in tests; nil means the real + // one. true means "the orphan is still there". A test can spawn a process that ignores + // SIGTERM, but it cannot spawn one the kernel refuses to reap, so the pid an abandon + // records there is alive only for as long as the stand-in child lives -- a wall-clock race + // no assertion should depend on. Set at construction and never written afterwards. + pidAliveOverride func(pid int) bool + // bootUnixOverride replaces the /proc/stat boot-time read in tests; nil means the real + // one. A test cannot reboot the host, so this is the only way to exercise the + // boot-generation check in loadAbandonMarker. Set at construction, never written after. + bootUnixOverride func() int64 + // aliveInterlockWaitOverride replaces daemonAliveInterlockWait in tests; zero means the + // production value. Without it, a test that deliberately wedges the interlock spends the + // whole production wait in wall clock. + aliveInterlockWaitOverride time.Duration + // removeDaemonFilesIO replaces the pid/info removal in removeDaemonFiles; nil means the + // real one. It is the only way to stand in for a BaseDir whose unlink never returns, which + // is what the deadline there exists for. Set at construction, never written after. + removeDaemonFilesIO func() + // procIdentityIO replaces the /proc read behind the orphan-identity probe; nil means the + // real one. It stands in for a read that stays parked in the kernel -- the case + // probeInFlight exists for, and one no test can produce against a real /proc file. Set at + // construction, never written after. + procIdentityIO func(pid int, start uint64) bool + // clearAbandonMarkerIO replaces the marker unlink in clearAbandonMarker; nil means the real + // one. Same purpose as removeDaemonFilesIO: only a seam can stand in for a removal that + // stays parked in the kernel past its deadline and lands after the abandon path has written + // the successor's marker. Set at construction, never written after. + clearAbandonMarkerIO func() error // statusMu serializes writes to the shared healthcheck status file: the // heartbeat loop and runOnce record ping outcomes concurrently, and @@ -156,14 +352,17 @@ func (d *daemon) run(ctx context.Context) int { }); err != nil { logging.Debug("daemon: write daemon info failed: %v", err) } - defer func() { - if err := health.RemoveDaemonPID(d.cfg.BaseDir); err != nil { - logging.Debug("daemon: remove pid file failed: %v", err) - } - if err := health.RemoveDaemonInfo(d.cfg.BaseDir); err != nil { - logging.Debug("daemon: remove daemon info failed: %v", err) - } - }() + defer d.removeDaemonFiles() + + // Inherit an abandon from a PREVIOUS process before anything can ping. The daemon exits + // after abandoning an unreapable child and systemd restarts it ten seconds later, so + // without this the very first beat of the new process -- heartbeatLoop beats once + // immediately, before its ticker -- would send a SUCCESS ping, flip the alive check back + // UP and fire a "recovered" alert, while the orphan still holds the backup lock and every + // scheduled run exits ExitBackupSkipped without pinging anything. Read regardless of + // HealthcheckEnabled so the state is carried (and cleanly cleared) even on a host that + // turns healthchecks on later. + d.loadAbandonMarker() if d.cfg.HealthcheckEnabled { if r := d.buildReporter(ctx); r != nil { @@ -171,11 +370,21 @@ func (d *daemon) run(ctx context.Context) int { } } + // loopCtx is how an ABANDONED run stops the background loops. They all return ONLY on + // their context being done, and on the abandon path the caller's context is still live -- + // no SIGTERM arrived, we are the ones deciding to die -- so without a cancel of our own + // the wg.Wait() below would block forever and the daemon would hang on the way OUT, + // trading one wedge for another. It also stops the heartbeat loop from pinging while we + // exit. Deferred as well as called explicitly so a panic unwind and go vet's lostcancel + // are both satisfied. + loopCtx, stopLoops := context.WithCancel(ctx) + defer stopLoops() + var wg sync.WaitGroup // The manual-outcome waker runs regardless of the heartbeat/update loops: it must receive // SIGUSR1 (so the default terminate action never fires) even when a piece of the healthcheck // wiring is off. processManualOutcome is itself a no-op when healthchecks are disabled or - // nothing was handed off. It returns on ctx.Done() and joins the waitgroup; it does NOT + // nothing was handed off. It returns on loopCtx.Done() and joins the waitgroup; it does NOT // signal.Stop -- that is owned by run()'s defer above so the disposition is reverted only after // the pidfile is gone (see the signal.Stop comment). wg.Add(1) @@ -183,10 +392,10 @@ func (d *daemon) run(ctx context.Context) int { defer wg.Done() for { select { - case <-ctx.Done(): + case <-loopCtx.Done(): return case <-usr1: - d.processManualOutcome(ctx) + d.processManualOutcome(loopCtx) } } }() @@ -195,17 +404,63 @@ func (d *daemon) run(ctx context.Context) int { wg.Add(1) go func() { defer wg.Done() - d.heartbeatLoop(ctx) + d.heartbeatLoop(loopCtx) }() wg.Add(1) go func() { defer wg.Done() - d.updateCheckLoop(ctx) + d.updateCheckLoop(loopCtx) }() } - d.scheduleLoop(ctx) - wg.Wait() + abandoned := d.scheduleLoop(loopCtx) + // Release the sibling loops BEFORE joining them: on the abandon path nothing else ever + // will. The abandon pings have already been sent by this point, under the still-live + // context, so cancelling here can never cut them short. + stopLoops() + if abandoned { + // A CAPPED join, unlike the clean stop below. The loops return as soon as their + // context is done, with one exception: a beat already inside buildReporter can be + // parked on the cross-process relay-secret flock, which honours neither a deadline nor + // the context. Everything worth transmitting is already transmitted, so a loop that + // cannot drain must not be allowed to hold the exit -- that would put the wedge back + // one step further out, with the daemon unable to die and systemd unable to restart + // it. The goroutines die with the process moments later. + if !waitGroupWithin(&wg, daemonLoopDrainWait) { + logging.Warning("daemon: background loops did not stop within %s; exiting anyway", daemonLoopDrainWait) + } + } else { + wg.Wait() + } + // Both deferred cleanups still run after either return below, in the same LIFO order: the + // pid file and .daemon_info.json go FIRST, signal.Stop LAST. So a restarting daemon never + // inherits a stale pid, and no standalone handoff can deliver SIGUSR1 to a process that + // has already reverted to the default-terminate disposition. + if abandoned { + // Restart=always + RestartSec=10 in the unit (buildDaemonUnit) brings a clean daemon + // back: no goroutine/fd accumulation behind a wedged child, and a live scheduler + // again. The orphan stays in D state until the kernel releases it; nothing in + // userspace can reap it. It is still in this unit's cgroup, so systemd's stop job + // will sit through its TimeoutStopSec phases waiting for a cgroup that cannot drain + // before the restart lands: the gap is minutes, not the nominal RestartSec=10. That is + // the accepted cost of keeping the default KillMode -- irrelevant for a once-daily + // scheduler, and the cgroup-wide kill is the only thing that collects the abandoned + // child's own descendants (see buildDaemonUnit). The restarted daemon picks the + // degrade back up from the marker abandonChild left on disk, so the alive check stays + // DOWN across the gap instead of re-greening. + // + // NOT ExitSuccess: an exit 0 is indistinguishable from an operator-requested stop. + // NOT ExitGenericError either -- docs/CLI_REFERENCE.md and docs/TROUBLESHOOTING.md + // publish 1 as one of the BENIGN non-zero codes (a run that succeeded with warnings), + // so a monitoring wrapper written from the documented contract would deliberately not + // alert on it. ExitBackupError is the documented "the backup operation failed" code, + // which is exactly what happened: no backup ran and none will until the host is + // cleared. No new constant is minted -- exit_codes_doc_drift_test.go requires a docs/ + // table row for every constant in internal/types/exit_codes.go. + logging.Error("ProxSave daemon exiting %d after abandoning an unreapable backup child; systemd will restart it", + types.ExitBackupError.Int()) + return types.ExitBackupError.Int() + } logging.Info("ProxSave daemon stopped") return types.ExitSuccess.Int() } @@ -267,14 +522,29 @@ func (d *daemon) processManualOutcome(ctx context.Context) { } d.recordOutcomePing(health.KindRunFinished, mo.ExitCode != 0, perr) + // A handed-off run that reached a real exit code is the same proof a supervised one is: a + // backup process took the lock the orphan used to hold and ran to completion. So it lifts an + // inherited abandon degrade for exactly the reason runOnce's does -- and this is the path + // that matters most, because `proxsave --backup` is how an operator PROVES the host they + // just fixed is working. Without it the backup check goes green off this handoff while the + // alive check keeps sending /fail, and the monitor reads "service dead, backups fine" until + // the next scheduled run. A SKIP proves the opposite (the lock is still held, or backups are + // off) and lifts nothing, mirroring runOnce exactly. + if mo.ExitCode != types.ExitBackupSkipped.Int() { + d.clearAbandonMarkerOnCompletedRun(fmt.Sprintf("a standalone backup completed (rid=%s exit=%d)", mo.RID, mo.ExitCode), mo.ExitCode) + } + if rmErr := health.RemoveManualOutcome(d.cfg.BaseDir); rmErr != nil { logging.Debug("daemon: remove manual outcome failed: %v", rmErr) } } -// scheduleLoop waits for the next daily run time and supervises a backup, until -// the context is cancelled. -func (d *daemon) scheduleLoop(ctx context.Context) { +// scheduleLoop waits for the next daily run time and supervises a backup, until the context +// is cancelled. It returns true when a run had to ABANDON a child the kernel will not let us +// reap: there is nothing useful to schedule behind such a child (it still holds the backup +// lock, so tomorrow's run would only exit ExitBackupSkipped), so the loop unwinds and lets +// run() exit for a systemd restart instead. +func (d *daemon) scheduleLoop(ctx context.Context) bool { for { next, err := cron.NextDaily(d.now(), d.cfg.SchedulerTime) if err != nil { @@ -291,9 +561,11 @@ func (d *daemon) scheduleLoop(ctx context.Context) { select { case <-ctx.Done(): timer.Stop() - return + return false case <-timer.C: - d.runOnce(ctx) + if d.runOnce(ctx) { + return true + } } } } @@ -301,9 +573,14 @@ func (d *daemon) scheduleLoop(ctx context.Context) { // runOnce launches ONE supervised backup as a child process under a hard timeout // and reports the outcome. A child that exceeds the budget is SIGTERM'd, then // SIGKILL'd, and reported as a hang. -func (d *daemon) runOnce(parentCtx context.Context) { +// +// It returns true ONLY when the child could not be REAPED even after that SIGKILL and had to +// be abandoned (see abandonChild): the caller must then unwind so the daemon exits and +// systemd restarts it. Every ordinary outcome -- success, failure, skip, shutdown, and an +// ordinary hang whose child actually died -- returns false and leaves the scheduler running. +func (d *daemon) runOnce(parentCtx context.Context) bool { if parentCtx.Err() != nil { // shutting down: do not start a run - return + return false } // Backups disabled: do NOT exec a child (it would exit 0 without backing up) // and do NOT ping an outcome, so the backup-outcome check honestly goes down @@ -311,7 +588,7 @@ func (d *daemon) runOnce(parentCtx context.Context) { // independent and keeps signalling the daemon is up. if !d.cfg.BackupEnabled { logging.Info("daemon: BACKUP_ENABLED=false; skipping the scheduled run (no outcome ping)") - return + return false } r := d.getReporter() rid := health.NewRunID() @@ -328,26 +605,39 @@ func (d *daemon) runOnce(parentCtx context.Context) { } cmd := d.buildBackupCmd(runCtx, tail, rid) cmd.Cancel = func() error { return cmd.Process.Signal(syscall.SIGTERM) } - cmd.WaitDelay = daemonKillGrace + cmd.WaitDelay = d.killGrace() logging.Info("daemon: launching backup (rid=%s timeout=%s)", rid, d.maxRunDuration()) - runErr := cmd.Run() - - // Interrupted by shutdown, not a real outcome: stay silent so we don't flip - // the check on a clean stop (the alive check going quiet signals the stop). - if parentCtx.Err() != nil { - return - } + reaped, runErr := d.superviseChild(runCtx, cmd) + // Read the captured tail ONCE, before branching: the abandon path wants it too. tailBuffer + // is mutex-guarded, so this is well defined even in the pathological case where an + // abandoned child's copy goroutine is still appending (os/exec has in fact already closed + // the parent pipe ends and drained those goroutines by then -- watchCtx does that when + // WaitDelay elapses, strictly before the reap deadline runs out). logBody := "" if tail != nil { logBody = tail.String() } + // The child is still running and can NEVER be waited on. This is the only branch that + // reports on a process that is not dead, and the only one that asks the daemon to exit. + // It must come first: every branch below assumes a finished process, and runErr is + // meaningless when the wait never completed. + if !reaped { + return d.abandonChild(parentCtx, r, cmd, rid, logBody) + } + + // Interrupted by shutdown, not a real outcome: stay silent so we don't flip + // the check on a clean stop (the alive check going quiet signals the stop). + if parentCtx.Err() != nil { + return false + } + if runCtx.Err() == context.DeadlineExceeded { logging.Error("daemon: backup exceeded %s and was killed (hang)", d.maxRunDuration()) d.reportBestEffort("hang", true, func() error { return d.hangPing(parentCtx, r, rid, logBody) }) - return + return false } code := exitCodeFromErr(runErr) @@ -357,15 +647,903 @@ func (d *daemon) runOnce(parentCtx context.Context) { // (F09-03). The real backup that holds the lock reports its own outcome. if code == types.ExitBackupSkipped.Int() { logging.Info("daemon: scheduled run skipped, no backup performed (rid=%s, no outcome ping)", rid) - return + return false } logging.Info("daemon: backup finished (rid=%s exit=%d)", rid, code) + // A run whose child actually RAN can lift an inherited abandon degrade: the daemon launched + // a child, it took the backup lock the orphan used to hold, and it was reaped. That is what + // the marker's claim is checked against -- not a skip (ExitBackupSkipped, handled above, is + // the signature of the orphan STILL holding the lock) and not a mere restart. Whether the + // backup itself succeeded is the backup check's business, not the alive check's. + // + // The gate is childReachedItsOwnExit and NOT the code, because `code` here is not always + // the child's: exitCodeFromErr synthesises 1 for an error that carries no wait status, and + // the loudest such error is a cmd.Start failure, which superviseChild returns as reaped=true + // (there is no child to abandon and no pid to leak). That daemon-side 1 is indistinguishable + // from the child's own "clean run, warnings only" 1 that exitProvesLockWasTaken must accept, + // so on the two branches with no pid to probe a fork/exec that never happened would delete + // the marker and hand the successor nothing -- the exact false GREEN this path exists to + // remove. A run with no child never reached the lock, so it answers nothing. + if childReachedItsOwnExit(runErr) { + d.clearAbandonMarkerOnCompletedRun("a supervised backup completed", code) + } else { + logging.Debug("daemon: the backup child never reached an exit status of its own (%v), so the run proves nothing about the backup lock; any abandoned-child marker is left alone", runErr) + } d.reportBestEffort("finish", code != 0, func() error { return d.finishPing(parentCtx, r, rid, code, logBody) }) // The child reached Phase-7 and wrote its per-channel notify outcomes; ping one // healthchecks check per channel it reported (Fase 2B / R4). Strictly after the child // exits, so it is naturally the last transmission of the run. d.reportNotifyOutcomes(parentCtx, r, rid) + return false +} + +// superviseChild starts cmd, waits for it, and gives up if the child cannot be reaped. +// It reports whether the wait completed at all and, when it did, the child's run error. +// +// It exists because cmd.Run() is NOT interruptible. os/exec's Cmd.Wait does +// "state, err := c.Process.Wait()" -- a wait4(2) -- BEFORE it ever reads the cancellation +// result, so cmd.Cancel and cmd.WaitDelay cannot unblock it: a child parked in +// TASK_UNINTERRUPTIBLE (D state -- a dead NFS/CIFS mount, a stuck device) never dequeues the +// SIGKILL os/exec sends after WaitDelay, is never reaped, and wait4 never returns. Waiting +// for Run() inline is therefore an UNBOUNDED wait on the scheduler's own goroutine: it wedges +// scheduleLoop so no later backup is ever scheduled, it keeps run() from reaching wg.Wait() +// so SIGTERM cannot stop the daemon, and it makes every branch downstream -- including the +// hang report written for exactly this case -- unreachable while the heartbeat loop keeps +// reporting the host green. +// +// So Wait runs on its own goroutine and this function bounds it. Start stays on the caller's +// goroutine: it is not the blocking part, and keeping it here means cmd.Process is published +// before the waiter exists, so the abandon path can log the orphan's pid without racing. +// +// Phase one is the normal case and is byte-for-byte today's behaviour, a Start failure +// included (reaped=true carrying the start error, which exitCodeFromErr still maps to 1). +// reaped=true therefore means "there is no orphan", NOT "a child ran": a caller that needs +// the second fact -- the abandon evidence rule does -- must ask childReachedItsOwnExit about +// the error, because the 1 alone cannot be told from a child's own warning exit. +// Phase two begins only once runCtx is done -- the exact instant os/exec begins the +// SIGTERM -> (WaitDelay) -> SIGKILL sequence, whether the watchdog budget expired or a +// shutdown cancelled the parent -- and gives the child reapWait() to actually be reaped. That +// anchor is what separates "did not die" from "died late but died": a child that ignored +// SIGTERM and only fell to the SIGKILL, or one whose pipes a grandchild held open for the +// full WaitDelay, is reaped inside that window and reported exactly as before. reaped=false +// means the kill grace AND the slack both elapsed and wait4 still has not returned: the +// child is abandoned, and the caller must tear the daemon down rather than pretend its slot +// is free. +// +// waitCh is buffered so the orphaned goroutine can never block on its send if the child is +// somehow reaped much later (the NFS server came back). That goroutine is deliberately dumb: +// its only statement is that send. It touches no daemon field, no reporter and no status +// file, so nothing it can reach is read after we give up and it cannot race the exit. +func (d *daemon) superviseChild(runCtx context.Context, cmd *exec.Cmd) (reaped bool, runErr error) { + if err := cmd.Start(); err != nil { + return true, err // never started: no child to abandon, no pid to leak + } + waitCh := make(chan error, 1) + go func() { waitCh <- cmd.Wait() }() + + select { + case err := <-waitCh: + return true, err + case <-runCtx.Done(): + } + + timer := time.NewTimer(d.reapWait()) + defer timer.Stop() + select { + case err := <-waitCh: + return true, err + case <-timer.C: + return false, nil + } +} + +// abandonChild gives up on a supervised child that survived SIGTERM followed by SIGKILL and +// that the kernel will not let os/exec reap, and reports whether the daemon must now exit. +// There is nothing left to wait for: the child cannot be killed, cannot be reaped, and holds +// its process slot (and the backup lock) until the kernel releases it. The pid stays behind +// in D state -- nothing in userspace can clear that -- but the daemon stops being hostage to +// it. +// +// The ordering is what the operator sees, and it is deliberate: the diagnosis lands in +// journald first; then the backup-outcome check goes DOWN -- this is the hang report that was +// structurally unreachable while cmd.Run() owned the wait -- then the marker is persisted so +// the state survives the exit, and only then the alive check goes DOWN. A monitor shown only +// one of those two checks lies about the host. +// +// The two TRANSMISSIONS are ordered ahead of the marker write on purpose. Persisting first +// reads better -- the state outlives everything after it -- but the marker write is local +// filesystem I/O against BaseDir, and the premise of this entire path is a host with wedged +// I/O. A BaseDir on that same dead mount would block those syscalls exactly as they blocked +// the child, and marker-first ordering would then cost BOTH monitoring signals (and runOnce's +// return with them). The pings are network calls the Reporter caps at pingTimeout; the marker +// write is bounded by persistAbandonMarker. Losing the marker costs the successor its +// inherited degrade -- bad, and logged -- while losing the pings costs the operator the +// outage itself. +// +// The backup /fail is also sent BEFORE aliveMu is touched. It is the primary signal of this +// whole change and has nothing to do with the alive check's ordering, so it must not be gated +// on another goroutine's lock. The alive silence latch then closes -- atomically, before the +// lock, so a beat that has not yet entered its critical section is dropped whatever happens +// next -- and only the degrade transmission itself takes aliveMu, with a deadline, purely to +// let a beat already on the wire land FIRST so the /fail is the last word on that check. +// +// A shutdown-time abandon returns false and PINGS nothing: the same rule as any other run +// interrupted by a stop (never flip a check on a clean stop), and the stop was requested, so +// there is no restart to ask for. It still writes the marker; see that branch. +func (d *daemon) abandonChild(parentCtx context.Context, r backupReporter, cmd *exec.Cmd, rid, logTail string) bool { + pid := -1 + if cmd.Process != nil { + pid = cmd.Process.Pid + } + // Read the child's start time NOW, while it is still unambiguously ours. It is what turns + // the pid in the marker into an identifier the successor can verify instead of a number the + // kernel may hand to something else (see abandonedChildGone). Bounded like every other read + // on this path, and best-effort: a 0 costs the successor the exact check, not the marker. + start := uint64(0) + if pid > 0 { + if v, answered := probeWithin(daemonProcProbeWait, func() uint64 { + ticks, ok := procStartTicks(pid) + if !ok { + return 0 + } + return ticks + }); answered { + start = v + } + if start == 0 { + logging.Debug("daemon: could not record the start time of abandoned child pid=%d; its successor will fall back to a cmdline match", pid) + } + } + note := fmt.Sprintf("backup child pid=%d (rid=%s) is unreapable; backups cannot run until the host clears the stuck I/O", pid, rid) + + if parentCtx.Err() != nil { + logging.Error("daemon: backup child pid=%d (rid=%s) survived SIGTERM + %s and cannot be reaped; leaving it behind and continuing the shutdown", + pid, rid, d.killGrace()) + // The silence rule is about TRANSMISSIONS -- no check may be flipped on a clean stop -- + // and the marker is not one: it is local state that changes nothing during this stop and + // only makes the NEXT process honest. Writing it here is what keeps `systemctl restart` + // (the reflex of any operator who notices a wedged daemon, and so the dominant way this + // branch is reached) from bringing the daemon back fully green over an orphan that still + // holds the backup lock, with every scheduled run exiting ExitBackupSkipped and pinging + // nothing. We already know the child is unreapable here; that fact is exactly what the + // successor needs. + d.persistAbandonMarker(pid, start, rid) + return false + } + + logging.Error("daemon: backup child pid=%d (rid=%s) exceeded %s, then survived SIGTERM and SIGKILL for %s: it is stuck in uninterruptible sleep and can never be reaped (look for a dead NFS/CIFS mount or a hung device)", + pid, rid, d.maxRunDuration(), d.reapWait()) + logging.Error("daemon: abandoning the run; the process stays behind in D state and keeps holding the backup lock until the host clears the stuck I/O") + + // The reporter runOnce captured may be stale by now: it is read once at the top of a run + // that may last MAX_RUN_DURATION, and in centralized mode beat's lazy re-resolve can + // install one behind our back (setReporter) at any point during it. Re-read it, or a + // daemon that started unpaired and got its URLs mid-run would silently drop BOTH mandated + // signals. No buildReporter fetch here -- see degradeAlive: this path must not block on the + // network, most likely against the same unreachable host that wedged the child. + if cur := d.getReporter(); cur != nil { + r = cur + } + + // The backup /fail goes out first, outside aliveMu and ahead of every local write: it is + // the report this whole path exists to make reachable, and neither another goroutine's lock + // nor a possibly-wedged disk may stand in front of it. Its own status-file record is + // deadline-bounded for that second reason (the ping itself has already been transmitted by + // the time that write is attempted). + d.reportBestEffortBounded("hang", true, daemonAbandonIOWait, func() error { return d.hangPing(parentCtx, r, rid, logTail) }) + + // Close the silence latch BEFORE writing the marker. It drops every beat that has not yet + // entered its critical section, and -- the reason it comes first -- it also stops + // reviewAbandonDegrade, which runs on that same beat, from deleting the marker written + // just below. A daemon that inherited a degrade whose old orphan has since died, and that + // then wedges on a NEW child, would otherwise have its fresh marker removed by a beat that + // passed the liveness probe moments earlier, and the successor would come up green over a + // live orphan. + d.aliveSilenced.Store(true) + + // Now persist, under a deadline. This is the only part that outlives the process, and it is + // what stops the daemon systemd is about to restart from re-greening the alive check with + // its immediate first beat. Best-effort: a write fault -- or a write that never returns -- + // must not cost us the degrade below. + d.persistAbandonMarker(pid, start, rid) + + // The lock only drains a beat ALREADY inside its transmission, so that its success ping + // cannot land behind the /fail below and re-green the check; that is worth a bounded wait + // and nothing more, hence the deadline. Failing to get it in time is logged and does not + // stop us: a runOnce that returns is worth more than a perfectly ordered pair of pings. + if d.lockAliveWithin(d.aliveInterlockWait()) { + defer d.aliveMu.Unlock() + } else { + logging.Warning("daemon: heartbeat interlock still held after %s; reporting the alive degrade anyway", d.aliveInterlockWait()) + } + d.degradeAlive(parentCtx, r, note) + return true +} + +// lockAliveWithin acquires aliveMu, giving up after limit. It reports whether the lock was taken +// (only then may the caller unlock). The wall clock is read directly, NOT through d.now: the +// tests freeze that clock, and a deadline that never advances is not a deadline. +func (d *daemon) lockAliveWithin(limit time.Duration) bool { + deadline := time.Now().Add(limit) + for { + if d.aliveMu.TryLock() { + return true + } + if !time.Now().Before(deadline) { + return false + } + time.Sleep(20 * time.Millisecond) + } +} + +// persistAbandonMarker records the abandon for the daemon systemd is about to start, under a +// deadline. +// +// The deadline is the point. health.WriteAbandon is os.MkdirAll + os.WriteFile + os.Rename +// against BaseDir, with no context and no timeout of its own (it is a stdlib-only sibling of +// the pid/status files by design), and the only reason this function is ever called is that +// the host has I/O the kernel will not let go of. A BaseDir on that filesystem -- a NAS-hosted +// BASE_DIR, or simply a wedged local disk -- turns those three syscalls into the same +// uninterruptible block that started all this, and an unbounded one here would stop runOnce +// from returning: the original wedge, reproduced one layer out, by the code written to remove +// it. runOnce RETURNING is the invariant; the marker is a best-effort nicety for the next +// process. So the write runs on its own goroutine and this waits a bounded time for it. +// +// A goroutine left behind on expiry is acceptable for the same reason it is in waitGroupWithin +// and superviseChild: every caller is on the abandon path, moments from exiting the process, +// and the goroutine touches nothing that is read afterwards. +// The start ticks come from the caller because they must be read while the child is still +// OURS -- see abandonChild. A 0 is honest and handled (the successor falls back to the cmdline +// identity test); a value read later, after the pid could already have been recycled, would +// not be. +// +// It takes abandonMu and latches abandonMarkerWritten BEFORE the write, so a concurrent clear +// either already ran (and this overwrites it) or sees the latch and leaves this file alone. +// Either order is correct; without the lock only one of them is. +func (d *daemon) persistAbandonMarker(pid int, start uint64, rid string) { + rec := health.AbandonRecord{PID: pid, Start: start, RID: rid, TS: d.now().Unix()} + d.abandonMu.Lock() + defer d.abandonMu.Unlock() + d.abandonMarkerWritten.Store(true) + // Publish the record with the latch, for the one reader that needs it: a removal that + // escaped abandonMu by staying parked in the kernel past its deadline and can still land on + // top of the write below. See restoreAbandonMarkerIfSuperseded. + d.abandonRec.Store(&rec) + if !runWithin(daemonAbandonIOWait, func() { + if err := health.WriteAbandon(d.cfg.BaseDir, rec); err != nil { + logging.Warning("daemon: could not persist the abandoned-child marker (%v); the alive check will re-green after the restart", err) + } + }) { + logging.Warning("daemon: writing the abandoned-child marker to %s did not complete within %s (is BASE_DIR on the wedged filesystem too?); continuing without it, so the alive check will re-green after the restart", + health.AbandonPath(d.cfg.BaseDir), daemonAbandonIOWait) + } +} + +// removeDaemonFiles clears the pid file and .daemon_info.json on the way out, under a +// deadline. +// +// The deadline is not decoration. This runs in run()'s LAST defer, so it is the final thing +// between the daemon and its exit -- and on the ABANDON path the exit is the whole point: +// maintainer decision, systemd Restart=always brings a clean daemon back. Both removals are +// plain os.Remove against BaseDir, which on that path may be the very filesystem that parked +// the child in D state; persistAbandonMarker goes to considerable lengths to bound its own +// writes for exactly this reason, and it would all be for nothing if the process then blocked +// here forever, unable to die and so never restarted. A removal that times out is left to the +// straggler goroutine (the process is about to be gone anyway) and the next daemon overwrites +// both files at startup regardless. +// +// removeDaemonFilesIO is the seam that lets a test wedge those two syscalls; nil means the +// real ones. +func (d *daemon) removeDaemonFiles() { + io := d.removeDaemonFilesIO + if io == nil { + io = func() { + if err := health.RemoveDaemonPID(d.cfg.BaseDir); err != nil { + logging.Debug("daemon: remove pid file failed: %v", err) + } + if err := health.RemoveDaemonInfo(d.cfg.BaseDir); err != nil { + logging.Debug("daemon: remove daemon info failed: %v", err) + } + } + } + if !runWithin(daemonAbandonIOWait, io) { + logging.Warning("daemon: clearing the pid file in %s did not complete within %s (is BASE_DIR on a wedged filesystem?); exiting anyway", + d.cfg.BaseDir, daemonAbandonIOWait) + } +} + +// runWithin runs fn on its own goroutine and waits up to limit for it to return, reporting +// whether it did. It bounds a call that has no context of its own -- local filesystem I/O on +// the abandon path -- and its callers must treat a false as "this may still be running": the +// goroutine is abandoned, not cancelled, because nothing in userspace can cancel a syscall +// parked in the kernel. Only ever used where the process is about to exit anyway. +func runWithin(limit time.Duration, fn func()) bool { + done := make(chan struct{}) + go func() { defer close(done); fn() }() + timer := time.NewTimer(limit) + defer timer.Stop() + select { + case <-done: + return true + case <-timer.C: + return false + } +} + +// waitGroupWithin joins wg, giving up after limit, and reports whether it drained. The helper +// goroutine outlives a timeout, which is only ever acceptable because the sole caller is on +// its way out of the process; it touches nothing else. +func waitGroupWithin(wg *sync.WaitGroup, limit time.Duration) bool { + done := make(chan struct{}) + go func() { wg.Wait(); close(done) }() + timer := time.NewTimer(limit) + defer timer.Stop() + select { + case <-done: + return true + case <-timer.C: + return false + } +} + +// loadAbandonMarker inherits an abandon left by a previous process (see abandonChild). It +// runs in run() before any loop starts, so the flags it sets are published to every reader. +// +// A marker is a CLAIM, not a verdict: "pid N is unreapable and is still holding the backup +// lock". This is the one place that claim can be checked against the host before a whole +// process lifetime is spent acting on it, so the two ways it can already be false are checked +// here -- backups administratively off, and the named orphan no longer on the host. Believing +// it unconditionally is how the fix for a false GREEN turns into a false RED on the check that +// pages people. +func (d *daemon) loadAbandonMarker() { + rec, err := health.ReadAbandon(d.cfg.BaseDir) + if err != nil { + // We could not read it, so we do not know whether an abandon happened: do NOT degrade on + // a guess. But a file we failed to read is still probably there, so remember that this + // process owes the identity dir a cleanup once a run proves the wedge is over -- see + // abandonMarkerOnDisk. + d.abandonMarkerOnDisk.Store(true) + logging.Debug("daemon: read abandoned-child marker failed: %v", err) + return + } + if rec == nil { + return + } + d.abandonMarkerOnDisk.Store(true) + + // The marker looks older than this boot, so the pid it names probably belongs to a process + // from a previous boot and cannot be our orphan: the pid space was recycled at boot. Retire + // it ahead of the BACKUP_ENABLED branch below -- a record from a previous boot must not be + // kept "for when backups are re-enabled" either, since re-enabling them cannot make a + // process that no longer exists relevant again. + // + // The comparison MAY NOT decide this on its own, and the gate on the identity probe is not + // belt and braces: /proc/stat btime is not a stamp the kernel recorded at boot, it is + // derived from the CURRENT realtime offset (getboottime64 = offs_real - offs_boot), so every + // forward step of the wall clock moves it forward by the same amount. A host that booted + // with a dead RTC, abandoned a child, and was then stepped forward by chrony has a btime + // later than a marker written minutes earlier during THIS boot -- and discarding it there + // re-greens the service-alive check over an orphan that is still wedged, which is the exact + // false GREEN this whole mechanism exists to remove. The same argument is why the identity + // half of the pid is a tick count and not a timestamp; see pidIsAbandonedChild and + // health.AbandonRecord.Start. + // + // So btime may only ever CONFIRM what the probe already says, never override it. What the + // branch is still worth is the ordering: it retires a pre-boot marker that the branch below + // would otherwise keep for a re-enable that can never make it true again. A boot time we + // cannot read (0) decides nothing. + if boot := d.bootUnix(); boot > 0 && rec.TS > 0 && rec.TS < boot && d.abandonedChildGone(rec.PID, rec.Start) { + logging.Info("daemon: discarding an abandoned-child marker for pid=%d (rid=%s) written before the current boot; that process is gone", rec.PID, rec.RID) + d.clearAbandonMarker("") + return + } + + // Backups are administratively OFF. The degrade says "backups are dead because an orphan + // holds the lock", but with BACKUP_ENABLED=false they are dead by operator decision, the + // backup-outcome check already goes honestly down on its own (runOnce pings nothing), and + // the alive check has nothing left to add. It also could never be lifted: runOnce returns + // at the BACKUP_ENABLED guard before any completed run can clear the marker, and a + // standalone backup refuses for the same reason (ExitBackupSkipped), so degrading here + // would pin the alive check DOWN forever on a daemon that is perfectly healthy -- and turning + // backups off is precisely what an operator does after reading the ERROR this path prints. + // BACKUP_ENABLED is read once at startup and the daemon restarts to pick up a config change, + // so this decision holds for the whole process lifetime. The marker is deliberately LEFT on + // disk: re-enabling backups makes it relevant again, and the first completed run clears it. + if !d.cfg.BackupEnabled { + logging.Info("daemon: a previous run abandoned backup child pid=%d (rid=%s), but BACKUP_ENABLED=false; not degrading the service-alive check (the backup check is down on its own). The marker is kept for when backups are re-enabled.", rec.PID, rec.RID) + return + } + + // The orphan is provably gone -- the host was rebooted, or the dead mount came back and the + // task finally died. kill(2) can only answer this in one direction (a pid that does not + // exist cannot be our unreapable child; a pid that does exist may be an unrelated process + // that reused the number after a reboot), so this only ever CLEARS, never invents, a + // degrade. Nothing here claims a backup succeeded: the backup-outcome check stays DOWN until + // a run actually reports one. + if d.abandonedChildGone(rec.PID, rec.Start) { + logging.Info("daemon: a previous run abandoned backup child pid=%d (rid=%s), but that process is gone; clearing the marker and reporting the service-alive check normally (the backup check stays down until a run reports one)", rec.PID, rec.RID) + d.clearAbandonMarker("") + return + } + + // Only NOW is the orphan's identity worth carrying: this is the one branch that acts on it + // for the rest of the process's life. The branches above RETIRED the marker, and a pid + // left in these fields by one of them would be probed by every completed run from here on + // -- after a reboot, against whatever unrelated process inherited the number -- and logged + // as "keeping the service-alive check DOWN" while the check is in fact green. + d.abandonPID = rec.PID + d.abandonStart = rec.Start + d.aliveDegraded.Store(true) + note := fmt.Sprintf("a previous run abandoned backup child pid=%d (rid=%s); no backup has completed since", rec.PID, rec.RID) + d.abandonNote.Store(¬e) + logging.Error("daemon: %s. The service-alive check is reported DOWN until a supervised backup completes. Check for a process stuck in D state (ps -eo pid,stat,wchan,cmd | grep ' D'), a dead NFS/CIFS mount, and for leftover backup helpers (tar/pigz/rclone) still writing to it.", note) +} + +// abandonedChildGone reports whether the child a previous process abandoned is no longer on +// this host. It answers about a PROCESS, not about a number: liveness alone is not an answer, +// because the kernel recycles pid numbers within a boot. +// +// That distinction is the whole safety of the mechanism in the OTHER direction. Every lift +// path -- the startup re-validation, the per-beat review, and a completed run -- is gated on +// this function, so a "still there" it gets wrong is not a transient annoyance: it pins the +// service-alive check DOWN on a host whose backups are running perfectly, with no run, no +// restart and no reboot able to lift it, until somebody deletes the marker by hand. And the +// coincidence is ordinary, not exotic: the mount heals, the orphan finally dies while nothing +// is watching (the daemon was stopped for the repair, or healthchecks are off so no beat ever +// reviews it), and the number is handed to the next long-lived process the host starts. The +// repo already refuses this oracle where the consequence was a misdirected signal; see +// probeProxsaveDaemonAlive ("the cmdline match is the SAFETY gate"). +// +// So: ESRCH from signal 0 proves the number is unused and the orphan is gone. Otherwise the +// number is in use by SOMETHING, and the identity check decides whether that something is +// still our child. Anything unreadable counts as GONE -- the process we are asking about is +// one we could observe in full detail when we abandoned it, so losing sight of it is evidence +// of a different process, and the direction that keeps the degrade liftable. +// +// A pid the marker never carried (0, or the -1 abandonChild records for a child that was +// never published) is unanswerable and counts as still there -- the conservative direction, +// which keeps the degrade. That case has its own lift rule; see +// clearAbandonMarkerOnCompletedRun. +func (d *daemon) abandonedChildGone(pid int, start uint64) bool { + if pid <= 0 { // never signal 0 or -1: those mean "my process group" and "every process" + return false + } + if d.pidAliveOverride != nil { + return !d.pidAliveOverride(pid) + } + if errors.Is(syscall.Kill(pid, 0), syscall.ESRCH) { + return true // the number is not in use at all + } + return !d.pidIsAbandonedChild(pid, start) +} + +// pidIsAbandonedChild reports whether the LIVE pid is still the process the marker named, +// under a deadline (see daemonProcProbeWait: expiry means the task is still parked in the +// kernel, which is itself "still there"). +// +// The exact test is the start time: (pid, starttime) is unique for the life of a boot, so a +// recycled number can never match, and a tick count cannot be moved by a clock step the way +// /proc/stat btime can. Markers written before that field existed carry 0, and fall back to +// the same cmdline test probeProxsaveDaemonAlive uses -- weaker (another proxsave --backup +// that happened to inherit the number would pass) but self-healing, because that process +// exits and the next beat lifts the degrade. +// +// At most ONE read is ever outstanding, and that is a requirement rather than an optimisation. +// probeWithin bounds the WAIT, not the read: a goroutine it gives up on is abandoned, not +// cancelled, and it is holding an open /proc file descriptor. Unlike every other caller of that +// helper this one is not on its way out of the process -- reviewAbandonDegrade runs it once per +// heartbeat for as long as the degrade stands -- so re-issuing a read that has already proved it +// can block would strand a goroutine and an fd every beat, indefinitely, on the one host an +// operator is actively investigating. While a read is still parked the answer is the same one +// its expiry gives ("still there"), and the next beat after it returns probes again. +func (d *daemon) pidIsAbandonedChild(pid int, start uint64) bool { + if !d.probeInFlight.CompareAndSwap(false, true) { + logging.Debug("daemon: an earlier identity probe for the abandoned child pid=%d is still parked in the kernel; treating it as still there without issuing another read", pid) + return true + } + ours, answered := probeWithin(daemonProcProbeWait, func() bool { + defer d.probeInFlight.Store(false) + if d.procIdentityIO != nil { + return d.procIdentityIO(pid, start) + } + if start > 0 { + cur, ok := procStartTicks(pid) + return ok && cur == start + } + return procIsBackupChild(pid) + }) + if !answered { + logging.Debug("daemon: identifying the abandoned child pid=%d did not complete within %s; treating it as still there", pid, daemonProcProbeWait) + return true + } + return ours +} + +// probeWithin runs fn on its own goroutine and waits up to limit for its answer, reporting +// whether one arrived. Same contract as runWithin -- a goroutine left behind on expiry is +// abandoned, not cancelled -- but it carries a value back, over a buffered channel so the +// straggler can never block on its send and never writes anything the caller reads. That last +// part is why fn RETURNS its result instead of assigning to a captured variable: an assignment +// from a goroutine that outlived the deadline would be a data race with the caller. +func probeWithin[T any](limit time.Duration, fn func() T) (result T, answered bool) { + ch := make(chan T, 1) + go func() { ch <- fn() }() + timer := time.NewTimer(limit) + defer timer.Stop() + select { + case v := <-ch: + return v, true + case <-timer.C: + var zero T + return zero, false + } +} + +// procStartTicks returns pid's start time in clock ticks since boot (/proc//stat field +// 22) and whether it could be read. The comm field (2) is wrapped in parentheses and may +// itself contain spaces AND parentheses, so the fields are counted from the LAST ')' -- the +// documented way to parse this file. +func procStartTicks(pid int) (uint64, bool) { + data, err := os.ReadFile("/proc/" + strconv.Itoa(pid) + "/stat") + if err != nil { + return 0, false + } + end := strings.LastIndexByte(string(data), ')') + if end < 0 { + return 0, false + } + // Fields after the comm start at field 3 (state), so field 22 is index 19 here. + fields := strings.Fields(string(data)[end+1:]) + if len(fields) < 20 { + return 0, false + } + v, err := strconv.ParseUint(fields[19], 10, 64) + if err != nil { + return 0, false + } + return v, true +} + +// procIsBackupChild reports whether pid's /proc//cmdline identifies a proxsave backup +// child: a "proxsave" token AND the exact "--backup" arg (buildBackupCmd's argv). Same shape +// as probeProxsaveDaemonAlive's daemon test, and used for the same reason -- a pid number on +// its own identifies nothing. Unreadable counts as NOT ours; see abandonedChildGone. +func procIsBackupChild(pid int) bool { + data, err := os.ReadFile("/proc/" + strconv.Itoa(pid) + "/cmdline") + if err != nil { + return false + } + // /proc//cmdline is NUL-separated argv; split to whole args so the match is on a real + // argument, not a coincidental "proxsave" substring buried inside an unrelated path. + args := strings.Split(string(data), "\x00") + var hasProxsave, hasBackup bool + for _, a := range args { + if strings.Contains(a, "proxsave") { + hasProxsave = true + } + if a == "--backup" { + hasBackup = true + } + } + return hasProxsave && hasBackup +} + +// reviewAbandonDegrade re-validates an inherited degrade against the host and lifts it once +// the orphan it names is gone. Called from beat, so a host that healed WITHOUT a restart (the +// NFS server came back, the D-state task finally died) stops reporting the alive check DOWN +// within one heartbeat interval instead of waiting up to a whole scheduling period for the +// next run to prove it -- the same false-RED-for-a-day the marker exists to avoid the inverse +// of. It costs one kill(2) plus one bounded /proc read per beat on a degraded daemon, and +// nothing at all otherwise. +func (d *daemon) reviewAbandonDegrade() { + // Cheap pre-check for the abandon path: once the latch is closed this process is on its + // way out after abandoning a child of its OWN, and the marker in the identity dir is the + // fresh one abandonChild just wrote for the successor -- not the inherited one this review + // is about. This is an optimisation, NOT the barrier: the latch and this load are separate + // atomics, so a beat that read it a moment too early is still on its way to the clear. The + // barrier that actually holds is in clearAbandonMarker (abandonMu + abandonMarkerWritten). + if d.aliveSilenced.Load() { + return + } + if !d.aliveDegraded.Load() || !d.abandonedChildGone(d.abandonPID, d.abandonStart) { + return + } + d.clearAbandonMarker(fmt.Sprintf("the abandoned backup child pid=%d is gone", d.abandonPID)) +} + +// clearAbandonMarkerOnCompletedRun retires the marker when a backup run reached a real exit +// code -- but only once the orphan the marker names is also gone. +// +// A completed run is NOT by itself proof that the wedge is over. The backup lock is checked +// LAST (internal/orchestrator/orchestrator.go, "4. Check lock file LAST"), after the +// directory, temp-dir, disk-space and permission gates, so a run can reach a real exit code +// without ever having reached the lock the orphan holds. Worse, the fault that wedges a child +// in D state -- a dead NFS/CIFS mount under the backup path -- is exactly the fault that fails +// those earlier gates. Believing the exit code alone therefore lifts the degrade precisely on +// the host that still needs it: the operator runs `proxsave --backup` to see what is wrong, it +// dies on the disk-space check with a non-zero code, and the alive check goes green over an +// orphan that has not moved. +// +// So a completed run only TRIGGERS the question, and the orphan probe answers it. Three cases, +// and which one applies is decided by whether a DEGRADE was ever raised -- not by the pid +// value, which says nothing on its own: +// +// - Degraded, and the marker named a pid. The probe decides, and only "the orphan is gone" +// lifts it. This is the case the paragraph above is about. +// - Degraded with NO usable pid -- a corrupt or truncated marker, which the health package +// deliberately reads as a real abandon (WriteAbandon does not fsync, and these hosts get +// hard-reset). There is nothing to probe, so the run's own evidence is all there is and +// only a code that proves the run REACHED the lock qualifies; see exitProvesLockWasTaken. +// - Nothing was degraded, but a file is still owed to the identity dir: one this process +// could not READ at startup (ReadAbandon's error branch), or one it deliberately KEPT +// because backups are administratively off. Neither is a free pass, and the SAME evidence +// rule applies. Both of those states are reached with a marker that may name a LIVE orphan +// -- the unreadable one most likely of all, since the host whose BaseDir I/O is wedged is +// exactly the host that wedges children -- and this process, having read nothing, knows +// even less about it than a degraded one does. Deleting on a pre-lock failure here is the +// same false GREEN as deleting on one above, only quieter: nothing is degraded in THIS +// process, so the whole cost lands on the successor, which inherits nothing and beats green +// over an orphan that still holds the backup lock. +// +// exitCode must be one a backup PROCESS actually reported. Callers hand it over unchanged +// (mo.ExitCode from the standalone handoff) or after checking that a child really exited +// (runOnce, via childReachedItsOwnExit): a code this daemon synthesised for a child that never +// ran is not evidence about anything, least of all about a lock that process never reached. +func (d *daemon) clearAbandonMarkerOnCompletedRun(reason string, exitCode int) { + if !d.aliveDegraded.Load() { + if !exitProvesLockWasTaken(exitCode) { + logging.Debug("daemon: %s, but the run did not reach the backup lock, so nothing proves the abandoned-child marker is stale; leaving it alone", reason) + return + } + d.clearAbandonMarker(reason) + return + } + if d.abandonPID > 0 { + if !d.abandonedChildGone(d.abandonPID, d.abandonStart) { + logging.Info("daemon: %s, but the abandoned backup child pid=%d is still on this host, so it never took the backup lock; keeping the service-alive check DOWN", reason, d.abandonPID) + // The degrade stands, but the reason it transmits must not: the note written at + // startup says "no backup has completed since", and one just did. Every beat from + // here on would repeat that as fact. Restate what is actually still true -- the + // orphan is on the host -- so the operator is told to hunt the D-state task rather + // than a backup failure that has already stopped happening. + d.setAbandonNote(fmt.Sprintf("a previous run abandoned backup child pid=%d, and that process is still on this host; backups have run since, but the orphan cannot be reaped", d.abandonPID)) + return + } + d.clearAbandonMarker(reason) + return + } + if !exitProvesLockWasTaken(exitCode) { + logging.Info("daemon: %s, but the marker names no pid this daemon can check and the run did not reach the backup lock, so nothing proves it was taken; keeping the service-alive check DOWN", reason) + return + } + d.clearAbandonMarker(reason) +} + +// setAbandonNote replaces the body the degraded beat transmits. See the abandonNote field. +func (d *daemon) setAbandonNote(note string) { + d.abandonNote.Store(¬e) +} + +// abandonNoteNow returns that body. A degrade always has one, but a nil is answered with a +// bare statement rather than an empty ping body: the beat must never transmit less than the +// fact that the check is down on purpose. +func (d *daemon) abandonNoteNow() string { + if n := d.abandonNote.Load(); n != nil { + return *n + } + return "a previous run abandoned an unreapable backup child" +} + +// exitProvesLockWasTaken reports whether a completed run's exit code is evidence that the run +// got as far as the BACKUP LOCK -- the LAST of the orchestrator's pre-flight gates -- and so +// that the orphan named by a marker is no longer holding it. +// +// Exactly the two documented NON-FAILURE codes qualify. 0 is a clean run. 1 is the same run +// with warnings: applyIssueExitCode (internal/orchestrator/extensions.go) promotes a clean run +// to ExitGenericError when it logged warnings or notify issues, and a run that raised real +// ERRORS becomes ExitBackupError instead, so a 1 is never a failure that was demoted into this +// set. docs/TROUBLESHOOTING.md documents both (the code table, and the "every backup reports +// warnings" section) -- including a routine state in which EVERY run on a perfectly healthy +// host exits 1, unacknowledged release notes after an upgrade. Refusing 1 here is therefore not +// caution: on such a host it is a service-alive DOWN that no run, no restart and no reboot can +// lift, which is the same unliftable false RED the pid identity check exists to prevent. +// +// Everything else is a failure code and proves nothing -- a pre-flight gate failure returns +// ExitBackupError (cmd/proxsave/backup_execution.go, runPreBackupChecks), and a dead mount +// under the backup path is exactly what fails those gates before the lock is ever reached. +// ExitBackupSkipped never arrives here: both callers filter it first. +func exitProvesLockWasTaken(exitCode int) bool { + return exitCode == types.ExitSuccess.Int() || exitCode == types.ExitGenericError.Int() +} + +// hostBootUnix reads the host's boot time from /proc/stat "btime", in Unix seconds. It +// returns 0 when the value cannot be read or parsed, which every caller must treat as +// "unknown" rather than as a boundary. +func hostBootUnix() int64 { + data, err := os.ReadFile("/proc/stat") + if err != nil { + return 0 + } + for _, line := range strings.Split(string(data), "\n") { + rest, ok := strings.CutPrefix(line, "btime ") + if !ok { + continue + } + v, err := strconv.ParseInt(strings.TrimSpace(rest), 10, 64) + if err != nil { + return 0 + } + return v + } + return 0 +} + +// bootUnix is hostBootUnix with the test seam applied. +func (d *daemon) bootUnix() int64 { + if d.bootUnixOverride != nil { + return d.bootUnixOverride() + } + return hostBootUnix() +} + +// clearAbandonMarker lifts an inherited degrade and removes the marker that carries it across +// restarts. reason names what proved the wedge was over and is logged only when a degrade was +// actually lifted (an empty reason logs nothing -- for callers that print their own line). +// +// The gate is abandonMarkerOnDisk, NOT the degrade: a process that could not read the marker +// never degraded but still has a file to delete, and leaving it would resurrect the degrade in +// the next process the moment the file became readable again. The CompareAndSwap keeps this a +// no-op, and silent, on the overwhelmingly common path where nothing was ever abandoned, so a +// normal run pays one atomic load and no syscall. +// +// The abandonMu / abandonMarkerWritten pair is the BARRIER against the abandon path, and it +// lives here rather than in any one caller because every caller needs it: the beat's review, +// runOnce's completed run, and processManualOutcome's SIGUSR1 handoff all reach this function, +// and the handoff's window is the widest of the three (abandonChild spends the whole interlock +// wait plus a ping after writing its marker, while the waker goroutine stays live until run() +// stops the loops). Once this process has written a marker for an orphan of its OWN, that file +// is the successor's inheritance and nothing here may delete it -- not even a caller that +// checked a latch a microsecond before abandonChild closed it and is only now arriving. The +// removal is deadline-bounded for the same reason the marker WRITE is: BaseDir may be on the +// filesystem that wedged the child, and an unbounded unlink under this lock would stall the +// abandon path that is waiting to persist. +// +// That deadline is also the one hole the lock cannot plug, and the removal closes it itself: +// runWithin gives up WAITING, it does not cancel the unlink, so on a wedged BaseDir -- the one +// host class this path exists for -- the lock is released with the syscall still queued, and it +// can land after abandonChild has persisted the successor's marker. See +// restoreAbandonMarkerIfSuperseded. For the same reason abandonMarkerOnDisk is only allowed to +// STAY down once the file is known to be gone: a removal that failed or timed out leaves a file +// nobody would ever retry, and it re-degrades the next daemon. +func (d *daemon) clearAbandonMarker(reason string) { + d.abandonMu.Lock() + defer d.abandonMu.Unlock() + if d.abandonMarkerWritten.Load() { + return + } + if !d.abandonMarkerOnDisk.CompareAndSwap(true, false) { + return + } + var cleared atomic.Bool + if !runWithin(daemonAbandonIOWait, func() { + if err := d.clearAbandonIO(); err != nil { + logging.Debug("daemon: clear abandoned-child marker failed: %v", err) + } else { + cleared.Store(true) + } + d.restoreAbandonMarkerIfSuperseded() + }) { + logging.Warning("daemon: removing the abandoned-child marker %s did not complete within %s (is BASE_DIR on a wedged filesystem?); the degrade is lifted in this process anyway", + health.AbandonPath(d.cfg.BaseDir), daemonAbandonIOWait) + } + if !cleared.Load() { + // The file may well still be there. Re-arm the gate so a later completed run retries the + // unlink instead of leaving a marker no caller in this process will ever touch again -- + // which the NEXT daemon reads, and degrades on, for a wedge that ended long ago. + d.abandonMarkerOnDisk.Store(true) + } + if d.aliveDegraded.CompareAndSwap(true, false) && reason != "" { + logging.Info("daemon: %s; the abandoned-child degrade is cleared and the alive check recovers on the next heartbeat", reason) + } +} + +// clearAbandonIO removes the marker file, with the test seam applied. +func (d *daemon) clearAbandonIO() error { + if d.clearAbandonMarkerIO != nil { + return d.clearAbandonMarkerIO() + } + return health.ClearAbandon(d.cfg.BaseDir) +} + +// restoreAbandonMarkerIfSuperseded puts back the marker abandonChild persisted for THIS +// process's own orphan, when the removal it runs at the tail of may have deleted it. +// +// It runs on clearAbandonMarker's bounded goroutine, and matters only when that goroutine +// outlived its deadline: the caller then returned and released abandonMu with the unlink still +// queued in the kernel, abandonChild acquired the lock and wrote the successor's marker, and +// this removal finally landed on top of it. The barrier cannot see that -- it guards the two +// WAITS, and nothing in userspace can recall a syscall the kernel is holding -- so the only +// remaining move is to notice afterwards and rewrite what was taken. Without it the successor +// daemon inherits nothing and beats green over a live orphan that still holds the backup lock. +// +// The record comes from an atomic pointer, not from under abandonMu, and that is deliberate: on +// the ordinary fast path this runs while its own caller still HOLDS that lock, so any acquire +// here would deadlock. A nil pointer -- no marker of our own was ever written -- is the fast +// path and the common case, and returns without touching anything. +func (d *daemon) restoreAbandonMarkerIfSuperseded() { + rec := d.abandonRec.Load() + if rec == nil { + return + } + if err := health.WriteAbandon(d.cfg.BaseDir, *rec); err != nil { + logging.Warning("daemon: an abandoned-child marker removal completed after the marker for pid=%d had been written, and restoring it failed (%v); the alive check will re-green after the restart", rec.PID, err) + return + } + logging.Warning("daemon: an abandoned-child marker removal completed after the marker for pid=%d had been written; the marker has been restored so the next daemon still inherits the degrade", rec.PID) +} + +// degradeAlive drives the SERVICE-ALIVE check DOWN on the way out of the process. +// +// It deliberately writes NOTHING to the local status file, unlike beat. The status file has +// exactly one field per kind that a reader consults for the alive sensor -- the KindHeartbeat +// record's TS and OK (health.sensorLevel, health.Diagnose) -- and a record written here would +// corrupt both: PingRecord.Down is read only for the backup and notify kinds, so a +// {OK:true, Down:true} heartbeat renders "ok"/green anyway, while its FRESH timestamp pushes +// the stale transition out by a whole heartbeat interval and keeps Diagnose answering +// DaemonUp=true -- for a daemon that is at that moment exiting. Worse, on a host with +// healthchecks disabled (heartbeatLoop never starts, so no heartbeat record has ever existed) +// it would fabricate the FIRST liveness record the file has ever held, flipping Diagnose from +// TxNoHeartbeat ("not running at all") to TxNotProvisioned ("up, not provisioned"). Writing +// nothing leaves the last real beat to age into TxStale, which is what actually happened. +// The DOWN signal rides the remote /fail, which is where the operator's alerting lives. +// +// There is no lazy centralized re-resolve here, unlike beat: the daemon is moments from +// exiting and must not block on a server fetch -- against, most likely, the same unreachable +// host -- ahead of the ping that matters. That is also why it must never be called with +// aliveMu held across a resolve; see the aliveMu field comment. +func (d *daemon) degradeAlive(ctx context.Context, r backupReporter, reason string) { + done := logging.DebugStart(d.logger, "hc ping", "kind=%s", health.KindHeartbeat) + var err error + if r == nil { + err = health.ErrNoAliveURL + } else { + err = r.AliveDegraded(ctx, reason+"; the daemon is abandoning it and restarting") + } + done(err) + if health.IsNoURLErr(err) { + logging.Debug("daemon: alive-degraded ping skipped (no url configured)") + } else if err != nil { + // err is already redacted by the Reporter (redactURLErr strips the url). + logging.Debug("daemon: alive-degraded ping failed: %v", err) + } +} + +// killGrace is how long a child gets between SIGTERM and SIGKILL (os/exec's Cmd.WaitDelay). +func (d *daemon) killGrace() time.Duration { + if d.killGraceOverride > 0 { + return d.killGraceOverride + } + return daemonKillGrace +} + +// reapWait is how long the daemon waits, from the moment the run context is done, for the +// child to be reaped before abandoning it. It covers the WHOLE SIGTERM -> SIGKILL grace plus +// daemonReapSlack, so it can never expire before the kill sequence it is waiting on has run +// its course; a child is only ever declared unreapable after the SIGKILL was actually sent. +func (d *daemon) reapWait() time.Duration { + if d.reapWaitOverride > 0 { + return d.reapWaitOverride + } + return d.killGrace() + daemonReapSlack +} + +// aliveInterlockWait is daemonAliveInterlockWait with the test seam applied. +func (d *daemon) aliveInterlockWait() time.Duration { + if d.aliveInterlockWaitOverride > 0 { + return d.aliveInterlockWaitOverride + } + return daemonAliveInterlockWait } // A nil reporter means no ping URL was ever resolved (unpaired/centralized, or the @@ -401,6 +1579,17 @@ func (d *daemon) hangPing(ctx context.Context, r backupReporter, rid, logTail st // failed ping). Every other result, success included, is a genuine transmission // attempt worth persisting so the run-side section can report the real state. func (d *daemon) reportBestEffort(label string, failed bool, fn func() error) { + d.reportBestEffortBounded(label, failed, 0, fn) +} + +// reportBestEffortBounded is reportBestEffort with a deadline on the local status-file WRITE +// only. recordLimit <= 0 means unbounded, which is what every ordinary run wants: the write is +// a few hundred microseconds, and a goroutine left parked on statusMu would accumulate on a +// daemon that stays up for months. The abandon path passes a limit because it is the one +// caller running on a host whose I/O is known to be wedged, and it must reach the alive +// degrade (and return) even if BaseDir is on the same dead mount as the child. The ping itself +// has already been transmitted by then, so a timeout here costs only the local record. +func (d *daemon) reportBestEffortBounded(label string, failed bool, recordLimit time.Duration, fn func() error) { done := logging.DebugStart(d.logger, "hc ping", "kind=%s", label) err := fn() done(err) @@ -414,7 +1603,14 @@ func (d *daemon) reportBestEffort(label string, failed bool, fn func() error) { } // label is already the kind ("start"/"hang"/"finish" == KindRun*). failed is the OUTCOME // signal (a failed finish / any hang) so the local sensor renders red, not green (F09-02). - d.recordOutcomePing(label, failed, err) + if recordLimit <= 0 { + d.recordOutcomePing(label, failed, err) + return + } + if !runWithin(recordLimit, func() { d.recordOutcomePing(label, failed, err) }) { + logging.Warning("daemon: recording the %s ping outcome did not complete within %s (is BASE_DIR on the wedged filesystem too?); the ping itself was sent", + label, recordLimit) + } } // recordPing persists one real transmission outcome to the shared status file, @@ -452,6 +1648,21 @@ func (d *daemon) heartbeatLoop(ctx context.Context) { } func (d *daemon) beat(ctx context.Context) { + // Silenced by an abandon in progress: the daemon is exiting and has already had the last + // word on this check. Drop the tick whole -- no ping, and no record either, so nothing + // here refreshes the liveness timestamp of a daemon that is dying. + if d.aliveSilenced.Load() { + return + } + // An inherited degrade is only as true as its premise. Re-check it before reporting it, so a + // host that healed without a restart recovers on this beat rather than on the next scheduled + // run. Outside aliveMu (it is a syscall on process state, not a transmission) and free on a + // daemon that inherited nothing. + d.reviewAbandonDegrade() + // Resolve the URLs BEFORE taking aliveMu. buildReporter can block indefinitely on the + // cross-process relay-secret flock, and aliveMu is on the abandon path: holding it across + // this would let an unrelated process wedge runOnce, which is the exact failure mode this + // whole change removes. Only the transmission below needs the lock. r := d.getReporter() if (r == nil || !r.HasAliveURL()) && d.cfg.HealthcheckMode == config.HealthcheckModeCentralized { if nr := d.buildReporter(ctx); nr != nil && nr.HasAliveURL() { @@ -459,6 +1670,16 @@ func (d *daemon) beat(ctx context.Context) { r = nr } } + // From here the hold is bounded: one ping (the reporter caps it at pingTimeout) plus one + // status-file write. It exists so an abandon cannot interleave its /fail with a success + // ping already on the wire and be re-greened by it; abandonChild waits for us, then + // transmits last. + d.aliveMu.Lock() + defer d.aliveMu.Unlock() + // Re-check under the lock: the abandon may have landed while we were resolving. + if d.aliveSilenced.Load() { + return + } done := logging.DebugStart(d.logger, "hc ping", "kind=%s", health.KindHeartbeat) // A nil reporter means no alive URL was ever resolved. Surface that as // ErrNoAliveURL (instead of returning) so the beat is STILL recorded -- as a @@ -467,9 +1688,17 @@ func (d *daemon) beat(ctx context.Context) { // (a heartbeat record exists, OK=false, reason no_url) from "daemon not running at // all" (no heartbeat record). A running daemon records its first beat immediately. var err error - if r == nil { + switch { + case r == nil: err = health.ErrNoAliveURL - } else { + case d.aliveDegraded.Load(): + // An abandon inherited from a previous process is still outstanding, so this beat + // reports the alive check DOWN instead of up: backups are dead and the check must say + // so until one completes. It is still RECORDED like any other beat below -- this + // daemon really is alive, and the local panel and health.Diagnose must keep saying so. + // Only the remote signal is inverted. + err = r.AliveDegraded(ctx, d.abandonNoteNow()) + default: err = r.Heartbeat(ctx) } done(err) @@ -1126,9 +2355,34 @@ func (d *daemon) setReporter(r backupReporter) { d.mu.Unlock() } +// childReachedItsOwnExit reports whether a supervised run's error describes a child that +// really ran and reached a wait status, so the code exitCodeFromErr derives from it is the +// CHILD's and not one this process invented. +// +// It exists because exitCodeFromErr is lossy in the one direction that matters to the abandon +// mechanism: it maps every error carrying no wait status to 1, which is also the child's own +// "succeeded with warnings" code. That is right for ALERTING -- a child that could not be +// forked is a real failure -- and wrong as EVIDENCE, because exitProvesLockWasTaken reads a 1 +// as "the run got past the backup lock". Only the caller still holds the error, so only the +// caller can tell the two apart; see runOnce. +// +// nil is the plain success. An *exec.ExitError is a child that ran and exited, whatever the +// code. Everything else -- a cmd.Start failure above all, but equally a pipe-copy fault that +// os/exec returns in place of the wait status -- means this process never saw the child reach +// an exit of its own, and answers nothing about the lock. That is the conservative direction: +// it keeps a marker, it never invents one. +func childReachedItsOwnExit(runErr error) bool { + if runErr == nil { + return true + } + var ee *exec.ExitError + return errors.As(runErr, &ee) +} + // exitCodeFromErr extracts a process exit code: 0 on success, the child's code on // a normal non-zero exit, and 1 when the child could not be started/run at all -// (which is a real failure worth alerting on). +// (which is a real failure worth alerting on). A caller that needs to know WHICH of those +// two a 1 is -- alerting does not, evidence does -- must ask childReachedItsOwnExit. func exitCodeFromErr(err error) int { if err == nil { return 0 diff --git a/cmd/proxsave/daemon_abandon_test.go b/cmd/proxsave/daemon_abandon_test.go new file mode 100644 index 00000000..41c2f5f1 --- /dev/null +++ b/cmd/proxsave/daemon_abandon_test.go @@ -0,0 +1,1709 @@ +// Package main contains the proxsave command entrypoint. +package main + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/tis24dev/proxsave/internal/health" + "github.com/tis24dev/proxsave/internal/types" +) + +// sigtermProofCmd builds a child that IGNORES SIGTERM, standing in for one parked in +// uninterruptible sleep. A real D-state task needs a wedged kernel path (a dead NFS mount), +// which no test can conjure in userspace, and SIGKILL always works on anything a test can +// spawn -- so the reap DEADLINE is what the tests drive, which is exactly the mechanism that +// was missing. From the daemon's side the two are indistinguishable: still there after the +// grace, with no exit status to report. +// +// The script must outlive the assertions but not the test binary; it exits on its own a few +// seconds later, so nothing is left running. +func sigtermProofCmd(seconds string) func(ctx context.Context) *exec.Cmd { + return shCmd(`trap "" TERM; sleep ` + seconds) +} + +// newAbandonDaemon builds a daemon whose child cannot be reaped inside the reap deadline. +// The kill grace is left at the PRODUCTION 30s on purpose: os/exec's own SIGKILL must not be +// able to land inside the test and reap the child, so what the test exercises is the daemon +// giving up, not the kernel rescuing it. +func newAbandonDaemon(t *testing.T, rep backupReporter, maxRun time.Duration) *daemon { + t.Helper() + d := newTestDaemon(t, rep, sigtermProofCmd("3"), maxRun) + d.reapWaitOverride = 200 * time.Millisecond + return d +} + +// TestRunOnceAbandonsUnreapableChild is the whole point of the fix. os/exec's Cmd.Wait calls +// Process.Wait (wait4) BEFORE it reads the cancellation result, so a child that never dies +// leaves cmd.Run() blocked forever: the scheduler wedges, SIGTERM cannot stop the daemon, and +// the hang report sitting downstream of cmd.Run() is unreachable for exactly the case it +// documents. runOnce must instead give up, report the backup check DOWN and the alive check +// DOWN, and ask the caller to take the daemon down for a systemd restart. +func TestRunOnceAbandonsUnreapableChild(t *testing.T) { + rep := &fakeReporter{alive: true, backupURL: true} + d := newAbandonDaemon(t, rep, 150*time.Millisecond) + + start := time.Now() + abandoned := d.runOnce(context.Background()) + elapsed := time.Since(start) + + if !abandoned { + t.Fatal("a child still unreaped past the deadline must be abandoned so the daemon can restart") + } + if elapsed > 5*time.Second { + t.Fatalf("runOnce stayed blocked on the unreapable child for %s", elapsed) + } + s := rep.snapshot() + if s.hung != 1 || s.finished != 0 { + t.Fatalf("abandoned run: hung=%d finished=%d, want 1/0", s.hung, s.finished) + } + if s.aliveDown != 1 { + t.Fatalf("abandoning must drive the alive check DOWN exactly once, got %d (alive must not stay green while backups are dead)", s.aliveDown) + } +} + +// TestRunOnceAbandonRecordsTheBackupOutcomeOnly pins what the LOCAL status file may and may +// not say after an abandon. The backup outcome is recorded DOWN, as for any hang. The alive +// side is NOT recorded, and that is deliberate: PingRecord.Down is read only for the backup +// and notify kinds (health.SensorRows), so a heartbeat record carrying it would still render +// green -- while its fresh TS would keep health.Diagnose answering DaemonUp=true, and push +// the stale transition a whole heartbeat interval into the future, for a daemon that is +// exiting. Writing nothing lets the last real beat age into TxStale, which is the truth. +func TestRunOnceAbandonRecordsTheBackupOutcomeOnly(t *testing.T) { + rep := &fakeReporter{alive: true, backupURL: true} + d := newAbandonDaemon(t, rep, 150*time.Millisecond) + + if !d.runOnce(context.Background()) { + t.Fatal("expected the run to be abandoned") + } + st, err := health.LoadStatus(d.cfg.BaseDir) + if err != nil { + t.Fatalf("LoadStatus: %v", err) + } + if hang := st.Record(health.KindRunHang); hang == nil || !hang.Down { + t.Fatalf("the backup-outcome record must be DOWN, got %+v", hang) + } + if hb := st.Record(health.KindHeartbeat); hb != nil { + t.Fatalf("the abandon must not write a liveness record for a daemon that is exiting, got %+v", hb) + } +} + +// TestAbandonWithNoReporterWritesNoPhantomHeartbeat guards the same rule where it does real +// damage. With healthchecks disabled the heartbeat loop never runs, so the status file holds +// NO heartbeat record at all and health.Diagnose correctly answers TxNoHeartbeat / "the +// daemon is not running". A record written by the abandon path for a ping that never left the +// process would be the first one the file ever held, flipping that verdict to +// TxNotProvisioned / DaemonUp=true at the exact moment the daemon dies -- and it would break +// the same no-phantom-ping invariant reportBestEffort already enforces for the outcome pings. +func TestAbandonWithNoReporterWritesNoPhantomHeartbeat(t *testing.T) { + d := newAbandonDaemon(t, nil, 150*time.Millisecond) + d.cfg.HealthcheckEnabled = false + + if !d.runOnce(context.Background()) { + t.Fatal("expected the run to be abandoned") + } + st, err := health.LoadStatus(d.cfg.BaseDir) + if err != nil { + t.Fatalf("LoadStatus: %v", err) + } + if hb := st.Record(health.KindHeartbeat); hb != nil { + t.Fatalf("a ping that never left the process must not be persisted, got %+v", hb) + } + if dg := health.Diagnose(st, 5*time.Minute, time.Now()); dg.DaemonUp { + t.Fatalf("the abandon must not fabricate liveness, got DaemonUp=true state=%v", dg.State) + } +} + +// TestRunOnceDoesNotAbandonAChildThatOnlyDiesOnSIGKILL is the other half of the contract, and +// the reason the reap deadline is anchored on the run context being done rather than on the +// watchdog budget. This child ignores SIGTERM and dies only when os/exec's WaitDelay SIGKILL +// lands -- late, but inside the window. It is reaped, so it is an ORDINARY hang: report it, +// leave the alive check alone, and keep the daemon running. +func TestRunOnceDoesNotAbandonAChildThatOnlyDiesOnSIGKILL(t *testing.T) { + rep := &fakeReporter{alive: true, backupURL: true} + d := newTestDaemon(t, rep, sigtermProofCmd("5"), 100*time.Millisecond) + d.killGraceOverride = 200 * time.Millisecond // SIGKILL lands at ~300ms + d.reapWaitOverride = 3 * time.Second // ...well inside the give-up window + + if d.runOnce(context.Background()) { + t.Fatal("a child that died late but DIED must not be abandoned (no daemon exit)") + } + s := rep.snapshot() + if s.hung != 1 || s.finished != 0 { + t.Fatalf("late-dying run: hung=%d finished=%d, want 1/0", s.hung, s.finished) + } + if s.aliveDown != 0 { + t.Fatalf("an ordinary hang must not degrade the alive check, got aliveDown=%d", s.aliveDown) + } +} + +// TestRunOnceOrdinaryHangDoesNotAbandon guards the blast radius on the common path, with the +// PRODUCTION graces: a child that overruns its budget and dies on the SIGTERM is still just a +// hang. It must not degrade the alive check and must not take the daemon down with it. +func TestRunOnceOrdinaryHangDoesNotAbandon(t *testing.T) { + rep := &fakeReporter{alive: true, backupURL: true} + d := newTestDaemon(t, rep, shCmd("sleep 5"), 150*time.Millisecond) + + if d.runOnce(context.Background()) { + t.Fatal("a hung child that actually died must not tear the daemon down") + } + s := rep.snapshot() + if s.hung != 1 || s.aliveDown != 0 { + t.Fatalf("ordinary hang: hung=%d aliveDown=%d, want 1/0", s.hung, s.aliveDown) + } +} + +// TestRunOnceAbandonDuringShutdownStaysSilent pins the asymmetry: when the abandon happens +// because we are already stopping, the existing silence rule wins -- no outcome ping may flip +// a check on a clean stop -- and no restart is requested. runOnce must still RETURN, which +// today it would not: it would sit in cmd.Wait until systemd's TimeoutStopSec SIGKILLed the +// daemon. +func TestRunOnceAbandonDuringShutdownStaysSilent(t *testing.T) { + rep := &fakeReporter{alive: true, backupURL: true} + d := newAbandonDaemon(t, rep, time.Hour) // an hour-long budget: only the cancel can end the run + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + time.AfterFunc(100*time.Millisecond, cancel) + + start := time.Now() + if d.runOnce(ctx) { + t.Fatal("a shutdown-time abandon must not request a restart; the daemon is already exiting") + } + // The give-up is ~300ms after the cancel. The bound is deliberately tight: a runOnce that + // only returns when the child happens to die anyway is the pre-fix behaviour, and there it + // is systemd's TimeoutStopSec, not the daemon, that ends the stop. + if elapsed := time.Since(start); elapsed > 1500*time.Millisecond { + t.Fatalf("runOnce did not give up on the unreapable child during shutdown (%s); the stop is at systemd's mercy", elapsed) + } + s := rep.snapshot() + if s.started != 1 { + t.Fatalf("the start ping fires before the child runs, got started=%d", s.started) + } + if s.hung != 0 || s.finished != 0 || s.aliveDown != 0 { + t.Fatalf("a stop must flip no check, got hung=%d finished=%d aliveDown=%d", s.hung, s.finished, s.aliveDown) + } +} + +// TestShutdownAbandonStillHandsTheDegradeToTheNextProcess covers the path an operator +// actually takes. Someone who notices a wedged daemon runs `systemctl restart`: the context is +// cancelled, the reap deadline expires, and the abandon happens on the SHUTDOWN branch -- which +// pings nothing, correctly, because no check may be flipped on a clean stop. But the marker is +// not a ping. It transmits nothing during the stop; it is the only way the process systemd +// starts ten seconds later learns that an unreapable orphan is still holding the backup lock. +// Without it that successor comes up fully green over dead backups, sends a SUCCESS heartbeat +// immediately, and every scheduled run exits ExitBackupSkipped without pinging anything -- the +// exact false green this whole change exists to remove, reached by its most common route. +func TestShutdownAbandonStillHandsTheDegradeToTheNextProcess(t *testing.T) { + rep := &fakeReporter{alive: true, backupURL: true} + d := newAbandonDaemon(t, rep, time.Hour) // only the cancel can end this run + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + time.AfterFunc(100*time.Millisecond, cancel) + + if d.runOnce(ctx) { + t.Fatal("a shutdown-time abandon must not request a restart") + } + if s := rep.snapshot(); s.hung != 0 || s.finished != 0 || s.aliveDown != 0 { + t.Fatalf("a stop must still flip no check, got hung=%d finished=%d aliveDown=%d", s.hung, s.finished, s.aliveDown) + } + + rec, err := health.ReadAbandon(d.cfg.BaseDir) + if err != nil { + t.Fatalf("ReadAbandon: %v", err) + } + if rec == nil { + t.Fatal("a shutdown-time abandon left no marker; the daemon systemd restarts comes up green over an orphan that still holds the backup lock") + } + if rec.PID <= 0 { + t.Fatalf("the marker must name the orphan an operator has to hunt for, got %+v", rec) + } + + next := &fakeReporter{alive: true, backupURL: true} + restarted := restartedDaemon(t, d, next) + if !restarted.aliveDegraded.Load() { + t.Fatal("the restarted daemon did not inherit the shutdown-time abandon") + } + restarted.beat(context.Background()) + if s := next.snapshot(); s.beats != 0 || s.aliveDown != 1 { + t.Fatalf("the successor must keep the alive check DOWN, got beats=%d aliveDown=%d", s.beats, s.aliveDown) + } +} + +// TestAbandonPingsTheBackupCheckBeforeWritingTheMarker pins the ordering the marker write must +// never invert. That write is os.MkdirAll + os.WriteFile + os.Rename against BaseDir, and this +// path only ever runs on a host with I/O the kernel will not let go of -- a BaseDir on that +// same dead mount blocks there exactly as the child did. Ordered first, it would cost both +// mandated signals; ordered after the /fail, the primary report is already on the wire. +func TestAbandonPingsTheBackupCheckBeforeWritingTheMarker(t *testing.T) { + rep := &blockingHangReporter{entered: make(chan struct{}), release: make(chan struct{})} + rep.alive, rep.backupURL = true, true + d := newAbandonDaemon(t, rep, 150*time.Millisecond) + + done := make(chan bool, 1) + go func() { done <- d.runOnce(context.Background()) }() + + select { + case <-rep.entered: + case <-time.After(10 * time.Second): + t.Fatal("the hang ping was never attempted") + } + // The /fail is on the wire and has not returned. Nothing local may have run before it. + if rec, err := health.ReadAbandon(d.cfg.BaseDir); err != nil || rec != nil { + t.Fatalf("the marker was written before the backup /fail was sent (rec=%+v err=%v); a wedged BaseDir would swallow the primary signal", rec, err) + } + close(rep.release) + + select { + case abandoned := <-done: + if !abandoned { + t.Fatal("expected the run to be abandoned") + } + case <-time.After(30 * time.Second): + t.Fatal("runOnce never returned") + } + if rec, err := health.ReadAbandon(d.cfg.BaseDir); err != nil || rec == nil { + t.Fatalf("the marker must still be written after the ping (rec=%+v err=%v)", rec, err) + } +} + +// blockingHangReporter parks the backup /fail inside its transmission until the test releases +// it, which is what makes "the ping is not gated on local disk" observable. +type blockingHangReporter struct { + fakeReporter + entered chan struct{} + release chan struct{} +} + +func (b *blockingHangReporter) RunHang(ctx context.Context, rid string, timeout time.Duration, tail string) error { + close(b.entered) + <-b.release + return b.fakeReporter.RunHang(ctx, rid, timeout, tail) +} + +// TestRunWithinAbandonsACallThatNeverReturns is the unit-level guarantee behind that ordering: +// a local write that never comes back must not be able to stop the abandon path. Nothing in +// userspace can cancel a syscall parked in the kernel, so the call is abandoned, not +// interrupted -- acceptable only because every caller is moments from exiting the process. +func TestRunWithinAbandonsACallThatNeverReturns(t *testing.T) { + release := make(chan struct{}) + defer close(release) + + start := time.Now() + if runWithin(150*time.Millisecond, func() { <-release }) { + t.Fatal("runWithin claimed a call that never returned had finished") + } + if elapsed := time.Since(start); elapsed > 3*time.Second { + t.Fatalf("runWithin waited %s on a wedged call; the deadline is not a deadline", elapsed) + } + if !runWithin(5*time.Second, func() {}) { + t.Fatal("runWithin must report a call that did finish") + } +} + +// TestBeatIsSuppressedWhileTheDaemonIsExiting pins the silence latch: heartbeatLoop is a live +// goroutine throughout the abandon, and one success beat from it would re-green the alive +// check before the process even exits. +func TestBeatIsSuppressedWhileTheDaemonIsExiting(t *testing.T) { + rep := &fakeReporter{alive: true} + d := newTestDaemon(t, rep, nil, time.Hour) + d.cfg.HealthcheckMode = "self" // no centralized rebuild, no network + d.aliveSilenced.Store(true) + + d.beat(context.Background()) + + if s := rep.snapshot(); s.beats != 0 || s.aliveDown != 0 { + t.Fatalf("an exiting daemon must transmit nothing more on the alive check, got beats=%d aliveDown=%d", s.beats, s.aliveDown) + } + st, err := health.LoadStatus(d.cfg.BaseDir) + if err != nil { + t.Fatalf("LoadStatus: %v", err) + } + if rec := st.Record(health.KindHeartbeat); rec != nil { + t.Fatalf("a suppressed beat must not record over the degraded one, got %+v", rec) + } +} + +// blockingBeatReporter parks the FIRST heartbeat inside its transmission until the test +// releases it, and logs the order in which the alive check's transmissions actually complete. +// It is what makes the latch's race observable: a bool read once at the top of beat() cannot +// order two concurrent POSTs, and the losing order is the one that re-greens a dead host. +type blockingBeatReporter struct { + fakeReporter + once sync.Once + entered chan struct{} // closed once a beat is inside Heartbeat, past the latch check + release chan struct{} // closed by the test to let that beat finish transmitting + + orderMu sync.Mutex + order []string +} + +func (b *blockingBeatReporter) Heartbeat(ctx context.Context) error { + b.once.Do(func() { + close(b.entered) + <-b.release + }) + b.note("beat") + return b.fakeReporter.Heartbeat(ctx) +} + +func (b *blockingBeatReporter) AliveDegraded(ctx context.Context, reason string) error { + b.note("degrade") + return b.fakeReporter.AliveDegraded(ctx, reason) +} + +func (b *blockingBeatReporter) note(what string) { + b.orderMu.Lock() + defer b.orderMu.Unlock() + b.order = append(b.order, what) +} + +func (b *blockingBeatReporter) transmissions() []string { + b.orderMu.Lock() + defer b.orderMu.Unlock() + return append([]string(nil), b.order...) +} + +// TestAbandonWaitsForAnInFlightBeat is the regression test for the race a bare latch cannot +// close. heartbeatLoop is live for the whole abandon, and beat() reads the latch ONCE and then +// spends up to a full ping timeout inside the transmission. A beat that entered before the +// latch closed must not be allowed to deliver its SUCCESS ping after the degrade's /fail: on +// the monitor that ping re-greens the alive check, and a green service over dead backups is +// the exact misreport the degrade exists to prevent. So the abandon has to WAIT for the +// in-flight beat and transmit last. +// +// The wait is on the ALIVE degrade only, and it is deadline-capped; the backup /fail has +// already gone out by this point (TestAbandonPingsTheBackupCheckBeforeTouchingTheAliveInterlock) +// and runOnce returns even if the lock never comes free +// (TestAbandonReportsEvenWhenTheHeartbeatInterlockIsStuck). +func TestAbandonWaitsForAnInFlightBeat(t *testing.T) { + rep := &blockingBeatReporter{entered: make(chan struct{}), release: make(chan struct{})} + rep.alive, rep.backupURL = true, true + d := newAbandonDaemon(t, rep, 150*time.Millisecond) + + beatDone := make(chan struct{}) + go func() { defer close(beatDone); d.beat(context.Background()) }() + <-rep.entered // the beat is now INSIDE the transmission, past the latch check + + abandonDone := make(chan bool, 1) + go func() { abandonDone <- d.runOnce(context.Background()) }() + + // runOnce reaches the abandon in ~350ms (budget + reap deadline). It must still not have + // reported: the beat it would be overtaken by is on the wire. + select { + case <-abandonDone: + t.Fatal("the abandon reported while a heartbeat was still in flight; that beat lands after the /fail and re-greens the alive check") + case <-time.After(900 * time.Millisecond): + } + + close(rep.release) + <-beatDone + select { + case abandoned := <-abandonDone: + if !abandoned { + t.Fatal("expected the run to be abandoned") + } + case <-time.After(10 * time.Second): + t.Fatal("the abandon never completed after the beat was released") + } + + got := rep.transmissions() + if len(got) == 0 || got[len(got)-1] != "degrade" { + t.Fatalf("the alive check's LAST transmission must be the degrade /fail, got %v", got) + } + if s := rep.snapshot(); s.beats != 1 || s.aliveDown != 1 { + t.Fatalf("beats=%d aliveDown=%d, want 1/1", s.beats, s.aliveDown) + } + // And the beat that was already on the wire must not have left a record behind either: + // the abandon writes no liveness record, so whatever the beat wrote is the last local + // word. It is a real beat, so it is allowed to stand -- but it must not be DOWN-flavoured + // noise from a suppressed tick. + st, err := health.LoadStatus(d.cfg.BaseDir) + if err != nil { + t.Fatalf("LoadStatus: %v", err) + } + if hb := st.Record(health.KindHeartbeat); hb == nil || hb.Down { + t.Fatalf("the in-flight beat's own record must stand unmodified, got %+v", hb) + } +} + +// TestBeatAfterDegradeIsDroppedWhole pairs with the test above: once the latch is closed under +// aliveMu, every later beat is dropped entirely -- no ping, and no record that would refresh +// the liveness timestamp of a daemon on its way out. +func TestBeatAfterDegradeIsDroppedWhole(t *testing.T) { + rep := &fakeReporter{alive: true, backupURL: true} + d := newAbandonDaemon(t, rep, 150*time.Millisecond) + + if !d.runOnce(context.Background()) { + t.Fatal("expected the run to be abandoned") + } + d.beat(context.Background()) + + if s := rep.snapshot(); s.beats != 0 { + t.Fatalf("a beat after the degrade must not re-green the alive check, got %d beats", s.beats) + } + st, err := health.LoadStatus(d.cfg.BaseDir) + if err != nil { + t.Fatalf("LoadStatus: %v", err) + } + if hb := st.Record(health.KindHeartbeat); hb != nil { + t.Fatalf("a suppressed beat must record nothing, got %+v", hb) + } +} + +// restartedDaemon is the daemon systemd brings back ten seconds after the abandoning one +// exited: a FRESH struct (runDaemon builds one per process, so every in-memory latch starts +// false) over the SAME BaseDir. It is the only way to observe what the second process +// actually reports, which is where the degrade either survives or is silently undone. +func restartedDaemon(t *testing.T, prev *daemon, rep backupReporter) *daemon { + t.Helper() + d := newTestDaemon(t, rep, nil, time.Hour) + d.cfg.BaseDir = prev.cfg.BaseDir + d.cfg.HealthcheckMode = "self" // no centralized rebuild, no network + // The orphan is still wedged -- the premise of every test that uses this helper. Pinned + // instead of left to the real kill(2) probe because the stand-in child a test can spawn + // exits on its own a few seconds in, which would otherwise make these assertions depend on + // how long the preceding lines happened to take. + d.pidAliveOverride = func(int) bool { return true } + d.loadAbandonMarker() // what run() does before any loop starts + return d +} + +// TestAbandonSurvivesTheRestart is the regression test for the window the in-memory latch +// cannot cover. The abandon exits the process on purpose (Restart=always / RestartSec=10), and +// heartbeatLoop beats IMMEDIATELY on start, before its ticker -- so an in-memory-only degrade +// is reversed about ten seconds after it was sent, flipping the alive check back UP and firing +// a "recovered" alert while the orphan still holds the backup lock and every scheduled run +// exits ExitBackupSkipped without pinging. The restarted daemon must inherit the degrade and +// keep reporting the alive check DOWN. +func TestAbandonSurvivesTheRestart(t *testing.T) { + first := &fakeReporter{alive: true, backupURL: true} + d := newAbandonDaemon(t, first, 150*time.Millisecond) + if !d.runOnce(context.Background()) { + t.Fatal("expected the run to be abandoned") + } + + second := &fakeReporter{alive: true, backupURL: true} + restarted := restartedDaemon(t, d, second) + if !restarted.aliveDegraded.Load() { + t.Fatal("the restarted daemon did not inherit the abandon; its first beat re-greens a host whose backups are dead") + } + restarted.beat(context.Background()) + + s := second.snapshot() + if s.beats != 0 { + t.Fatalf("the restarted daemon sent %d success heartbeat(s); alive must not go green while backups are dead", s.beats) + } + if s.aliveDown != 1 { + t.Fatalf("the restarted daemon must keep reporting the alive check DOWN, got aliveDown=%d", s.aliveDown) + } + // ...and it must still record its OWN liveness locally: this daemon really is running, and + // the run-side panel / health.Diagnose read that record. Only the REMOTE signal is + // inverted; suppressing the record too would make a live daemon look dead locally. + st, err := health.LoadStatus(restarted.cfg.BaseDir) + if err != nil { + t.Fatalf("LoadStatus: %v", err) + } + if hb := st.Record(health.KindHeartbeat); hb == nil || !hb.OK { + t.Fatalf("a degraded but running daemon must still record liveness, got %+v", hb) + } + if dg := health.Diagnose(st, 5*time.Minute, time.Now()); !dg.DaemonUp { + t.Fatalf("the restarted daemon is up and must diagnose as up, got state=%v", dg.State) + } +} + +// TestCompletedRunClearsTheInheritedDegrade is the other half: the degrade must not be +// permanent, or the alive check is red forever on a host the operator has already fixed. A run +// that reaches an exit code proves the orphan no longer holds the backup lock and the daemon +// can supervise a child again, so it lifts the degrade -- marker and all, so a further restart +// does not resurrect it. A SKIP does not, because a skip is the signature of the orphan still +// holding that lock. +func TestCompletedRunClearsTheInheritedDegrade(t *testing.T) { + first := &fakeReporter{alive: true, backupURL: true} + d := newAbandonDaemon(t, first, 150*time.Millisecond) + if !d.runOnce(context.Background()) { + t.Fatal("expected the run to be abandoned") + } + + skipped := restartedDaemon(t, d, &fakeReporter{alive: true, backupURL: true}) + // The orphan died after this daemon inherited the degrade and before any beat could review + // it, so the probe now says "gone" and the skip rule is the ONLY thing left standing between + // this run and a lift. Without that, the assertion below passes on the probe alone and stops + // testing the rule it names. + skipped.pidAliveOverride = func(int) bool { return false } + skipped.newBackupCmd = shCmd("exit " + strconv.Itoa(types.ExitBackupSkipped.Int())) + skipped.runOnce(context.Background()) + if !skipped.aliveDegraded.Load() { + t.Fatal("a skipped run performed no backup and proves nothing; only a run that reached the lock may lift the degrade") + } + + rep := &fakeReporter{alive: true, backupURL: true} + recovered := restartedDaemon(t, d, rep) + // The host healed while this daemon was up: the mount came back and the orphan finally + // died. Only then can a completed run prove the wedge is over -- the exit code alone does + // not, because the backup lock is checked after the pre-flight gates a dead mount fails. + // See clearAbandonMarkerOnCompletedRun. + recovered.pidAliveOverride = func(int) bool { return false } + recovered.newBackupCmd = shCmd("exit 0") + recovered.runOnce(context.Background()) + + if recovered.aliveDegraded.Load() { + t.Fatal("a completed backup must lift the degrade, or the alive check stays red on a healed host") + } + if rec, err := health.ReadAbandon(recovered.cfg.BaseDir); err != nil || rec != nil { + t.Fatalf("the marker must be gone so a later restart does not resurrect the degrade (rec=%+v err=%v)", rec, err) + } + recovered.beat(context.Background()) + if s := rep.snapshot(); s.beats != 1 || s.aliveDown != 0 { + t.Fatalf("after recovery the beat must be a plain success ping, got beats=%d aliveDown=%d", s.beats, s.aliveDown) + } +} + +// TestStandaloneBackupLiftsTheInheritedDegrade covers the operator's most natural remediation. +// After fixing the host they PROVE it with `proxsave --backup`, whose outcome is handed off by +// SIGUSR1 and pinged by processManualOutcome -- the same finish machinery a supervised run +// uses. If that path does not lift the degrade, the backup check goes GREEN off the handoff +// while the alive check keeps sending /fail, and the monitor reads "service dead, backups +// fine" until the next scheduled run, up to a full scheduling period away. +func TestStandaloneBackupLiftsTheInheritedDegrade(t *testing.T) { + first := &fakeReporter{alive: true, backupURL: true} + d := newAbandonDaemon(t, first, 150*time.Millisecond) + if !d.runOnce(context.Background()) { + t.Fatal("expected the run to be abandoned") + } + + rep := &fakeReporter{alive: true, backupURL: true} + restarted := restartedDaemon(t, d, rep) + restarted.cfg.HealthcheckEnabled = true + if !restarted.aliveDegraded.Load() { + t.Fatal("the restarted daemon must inherit the degrade") + } + + // The operator fixed the host first: the orphan is gone by the time they prove it with a + // backup. Without that, a non-zero exit from a pre-lock gate would be enough to lift the + // degrade -- see TestCompletedRunOverALiveOrphanKeepsTheDegrade. + restarted.pidAliveOverride = func(int) bool { return false } + + if err := health.WriteManualOutcome(restarted.cfg.BaseDir, "rid-manual", time.Now().Unix(), 0); err != nil { + t.Fatalf("WriteManualOutcome: %v", err) + } + restarted.processManualOutcome(context.Background()) + + if restarted.aliveDegraded.Load() { + t.Fatal("a completed standalone backup must lift the degrade; otherwise the monitor says 'service dead, backups fine'") + } + if rec, err := health.ReadAbandon(restarted.cfg.BaseDir); err != nil || rec != nil { + t.Fatalf("the marker must be gone so a later restart does not resurrect the degrade (rec=%+v err=%v)", rec, err) + } + restarted.beat(context.Background()) + if s := rep.snapshot(); s.beats != 1 || s.aliveDown != 0 { + t.Fatalf("after the handoff the beat must be a plain success ping, got beats=%d aliveDown=%d", s.beats, s.aliveDown) + } +} + +// TestCompletedRunOverALiveOrphanKeepsTheDegrade is the counterweight to +// TestCompletedRunClearsTheInheritedDegrade, and it covers the gap three rounds of review +// walked past. +// +// A run reaching a real exit code does NOT prove it took the backup lock. The lock is checked +// LAST (internal/orchestrator/orchestrator.go, "4. Check lock file LAST"), after the +// directory, temp-dir, disk-space and permission gates -- and a dead NFS/CIFS mount, the very +// fault that parks a child in D state, is what fails those gates first. So the operator's +// `proxsave --backup`, run precisely to see what is wrong, dies on the disk-space check with a +// non-zero code without ever reaching the orphan's lock. If that lifted the degrade, the alive +// check would go GREEN over an orphan that has not moved -- the "recovered" alert this whole +// mechanism exists to prevent. +func TestCompletedRunOverALiveOrphanKeepsTheDegrade(t *testing.T) { + first := &fakeReporter{alive: true, backupURL: true} + d := newAbandonDaemon(t, first, 150*time.Millisecond) + if !d.runOnce(context.Background()) { + t.Fatal("expected the run to be abandoned") + } + + // Supervised half. restartedDaemon pins the orphan as still wedged, which is the premise. + rep := &fakeReporter{alive: true, backupURL: true} + restarted := restartedDaemon(t, d, rep) + restarted.newBackupCmd = shCmd("exit " + strconv.Itoa(types.ExitBackupError.Int())) + restarted.runOnce(context.Background()) + + if !restarted.aliveDegraded.Load() { + t.Fatal("a run that failed before the lock check proves nothing about the orphan; the degrade must stand") + } + if rec, err := health.ReadAbandon(restarted.cfg.BaseDir); err != nil || rec == nil { + t.Fatalf("the marker must survive so a later restart still inherits the degrade (rec=%+v err=%v)", rec, err) + } + + // Standalone half: the same rule through the SIGUSR1 handoff. + manualRep := &fakeReporter{alive: true, backupURL: true} + manual := restartedDaemon(t, d, manualRep) + manual.cfg.HealthcheckEnabled = true + if err := health.WriteManualOutcome(manual.cfg.BaseDir, "rid-early-fail", time.Now().Unix(), types.ExitBackupError.Int()); err != nil { + t.Fatalf("WriteManualOutcome: %v", err) + } + manual.processManualOutcome(context.Background()) + + if !manual.aliveDegraded.Load() { + t.Fatal("a standalone run that failed before the lock check must not lift the degrade either") + } +} + +// TestTheKeptDegradeStopsClaimingNoBackupHasCompleted covers the BODY of the signal the test +// above pins the existence of. +// +// Keeping the degrade over a live orphan is right, and it is also the one state in which the +// note written at startup ages into a falsehood: it says "no backup has completed since", the +// beat repeats it verbatim on every heartbeat, and by then a supervised run has completed. The +// operator reading that alert is sent hunting a backup failure that has already stopped +// happening instead of the D-state task that is actually holding the check down. The degrade +// survives here for the orphan's sake, so the orphan is what it must say. +func TestTheKeptDegradeStopsClaimingNoBackupHasCompleted(t *testing.T) { + first := &fakeReporter{alive: true, backupURL: true} + d := newAbandonDaemon(t, first, 150*time.Millisecond) + if !d.runOnce(context.Background()) { + t.Fatal("expected the run to be abandoned") + } + + rep := &fakeReporter{alive: true, backupURL: true} + restarted := restartedDaemon(t, d, rep) + restarted.cfg.HealthcheckEnabled = true + + restarted.beat(context.Background()) + if s := rep.snapshot(); !strings.Contains(s.lastAliveReason, "no backup has completed since") { + t.Fatalf("before any run the inherited note is the honest one, got %q", s.lastAliveReason) + } + + // A backup now completes end to end while the orphan is still parked in D state -- the + // mount healed, the task did not. The degrade stands (the orphan may still hold its lock), + // but the claim that no backup has completed does not. + restarted.newBackupCmd = shCmd("exit 0") + restarted.runOnce(context.Background()) + if !restarted.aliveDegraded.Load() { + t.Fatal("the orphan is still on the host; the degrade must stand") + } + + restarted.beat(context.Background()) + s := rep.snapshot() + if strings.Contains(s.lastAliveReason, "no backup has completed since") { + t.Fatalf("the alive /fail still tells the operator no backup has completed, after one did: %q", s.lastAliveReason) + } + if !strings.Contains(s.lastAliveReason, "still on this host") { + t.Fatalf("the body must name what is actually still true -- the unreapable orphan -- got %q", s.lastAliveReason) + } +} + +// TestMarkerFromBeforeThisBootIsDiscarded covers what the boot-generation check is worth once +// the (pid, starttime) probe answers the recycled-number case on its own: ORDERING. A record +// from a previous boot must not be kept "for when backups are re-enabled", because re-enabling +// backups cannot make a process that no longer exists relevant again -- and the BACKUP_ENABLED +// branch, which sits below this one, would keep it on a daemon that is otherwise healthy. +// +// It runs the REAL probe against a pid the test already reaped, so btime is only ever +// CONFIRMING what the identity check independently says. A btime that DISAGREED with a live +// orphan may not retire anything; see TestAForwardClockStepMayNotRetireALiveOrphansMarker. +func TestMarkerFromBeforeThisBootIsDiscarded(t *testing.T) { + base := t.TempDir() + reaped := exec.Command("/bin/sh", "-c", "exit 0") + if err := reaped.Run(); err != nil { + t.Fatalf("run the throwaway child: %v", err) + } + dead := reaped.Process.Pid // exited AND waited on: this pid is gone + + if err := health.WriteAbandon(base, health.AbandonRecord{PID: dead, RID: "rid-old", TS: time.Now().Unix()}); err != nil { + t.Fatalf("WriteAbandon: %v", err) + } + rep := &fakeReporter{alive: true, backupURL: true} + rebooted := newTestDaemon(t, rep, nil, time.Hour) + rebooted.cfg.BaseDir = base + rebooted.cfg.HealthcheckMode = "self" + // Backups were turned off after the abandon -- the reaction the ERROR that path prints + // invites -- so nothing below the boot check would ever retire this record. + rebooted.cfg.BackupEnabled = false + rebooted.bootUnixOverride = func() int64 { return time.Now().Add(time.Hour).Unix() } + rebooted.loadAbandonMarker() + + if rebooted.aliveDegraded.Load() { + t.Fatal("a marker written before the current boot names a process that cannot still exist; it must not degrade") + } + if rec, err := health.ReadAbandon(base); err != nil || rec != nil { + t.Fatalf("the stale marker must be removed, not left to be re-read forever (rec=%+v err=%v)", rec, err) + } +} + +// TestAForwardClockStepMayNotRetireALiveOrphansMarker is the counterweight to the test above, +// and it covers the false GREEN a wall-clock oracle creates on its own. +// +// /proc/stat btime is not a stamp the kernel recorded at boot: it derives it from the CURRENT +// realtime offset (getboottime64 = offs_real - offs_boot), so every forward step of the wall +// clock moves btime forward by exactly the same amount. A host that booted with a dead RTC, +// abandoned a wedged child, and was then stepped forward by chrony therefore has a btime LATER +// than a marker written minutes earlier during that very boot. Retiring it there re-greens the +// service-alive check over an orphan that is still holding the backup lock -- and the marker's +// own (pid, starttime) pair, which no clock step can move, says so at the same moment. +// +// This runs the real probe, over a pid that is unquestionably alive and whose start time was +// recorded honestly: the test process itself. +func TestAForwardClockStepMayNotRetireALiveOrphansMarker(t *testing.T) { + base := t.TempDir() + self, ok := procStartTicks(os.Getpid()) + if !ok { + t.Fatalf("procStartTicks(self) failed; the identity probe cannot be exercised at all") + } + if err := health.WriteAbandon(base, health.AbandonRecord{ + PID: os.Getpid(), Start: self, RID: "rid-old", TS: time.Now().Unix(), + }); err != nil { + t.Fatalf("WriteAbandon: %v", err) + } + + rep := &fakeReporter{alive: true, backupURL: true} + d := newTestDaemon(t, rep, nil, time.Hour) + d.cfg.BaseDir = base + d.cfg.HealthcheckMode = "self" + // The clock was stepped forward after the marker was written, so btime now looks later than + // a record from this same boot. No pidAliveOverride: the point is that the real probe's + // answer must win. + d.bootUnixOverride = func() int64 { return time.Now().Add(time.Hour).Unix() } + d.loadAbandonMarker() + + if !d.aliveDegraded.Load() { + t.Fatal("the marker's own (pid, starttime) pair still names a live process on this host; a wall-clock comparison must not overrule it and re-green the alive check over a live orphan") + } + if rec, err := health.ReadAbandon(base); err != nil || rec == nil { + t.Fatalf("the marker must survive so the successor still inherits the degrade (rec=%+v err=%v)", rec, err) + } + d.beat(context.Background()) + if s := rep.snapshot(); s.beats != 0 || s.aliveDown != 1 { + t.Fatalf("the beat must keep reporting the alive check DOWN, got beats=%d aliveDown=%d", s.beats, s.aliveDown) + } +} + +// TestTheOrphanProbeRecognisesALiveProcess is the positive control for the one branch that +// enforces the whole mechanism: "the orphan is STILL THERE". +// +// Every other test that needs a live orphan pins pidAliveOverride, which replaces the probe +// entirely, and both tests that reach the real code assert the GONE answer -- so a defect in the +// field-22 parse or in the identity gate would turn every live orphan into "gone", lift the +// degrade, delete the marker and re-green proxsave-alive over a wedged child, with the suite +// still passing. The subject is a pid that is unquestionably alive and whose start time was read +// honestly: the test process itself. +func TestTheOrphanProbeRecognisesALiveProcess(t *testing.T) { + self, ok := procStartTicks(os.Getpid()) + if !ok || self == 0 { + t.Fatalf("procStartTicks(self) = %d ok=%v; /proc//stat field 22 is not being read", self, ok) + } + // Field 22 is monotonic in start order and the fields around it are not, so bracketing the + // test process between pid 1 and a child started right now catches an off-by-one that lands + // on a plausible-looking number in either direction. + if init, iok := procStartTicks(1); iok && self <= init { + t.Fatalf("procStartTicks: self=%d is not later than pid 1's %d; the wrong /proc//stat field is being read", self, init) + } + later := exec.Command("/bin/sh", "-c", "sleep 30") + if err := later.Start(); err != nil { + t.Fatalf("start the younger child: %v", err) + } + defer func() { + _ = later.Process.Kill() + _ = later.Wait() + }() + child, ok := procStartTicks(later.Process.Pid) + if !ok || child <= self { + t.Fatalf("procStartTicks: a child started just now reads %d (ok=%v), not later than this process's %d; the wrong /proc//stat field is being read", child, ok, self) + } + + d := newTestDaemon(t, nil, nil, time.Hour) + // The real probe over a live process that is emphatically not the test binary. + if d.abandonedChildGone(later.Process.Pid, child) { + t.Fatal("a child this test just started is on the host; reporting it GONE lifts every degrade over a wedged orphan") + } + if d.abandonedChildGone(os.Getpid(), self) { + t.Fatal("a LIVE pid whose recorded start time matches must be reported STILL THERE, or every degrade is lifted over a child that never moved") + } + if !d.abandonedChildGone(os.Getpid(), self+1) { + t.Fatal("a live pid whose start time does NOT match is a different process; calling it our child pins the alive check DOWN on a healed host") + } +} + +// TestAbandonRecordsTheOrphansRealStartTime pins the round trip the successor's identity check +// is built on. A marker that carries a zero start time still works, but only through the weaker +// cmdline fallback -- so a silent regression in the read at abandon time downgrades every later +// check without failing anything. +func TestAbandonRecordsTheOrphansRealStartTime(t *testing.T) { + rep := &fakeReporter{alive: true, backupURL: true} + d := newAbandonDaemon(t, rep, 150*time.Millisecond) + if !d.runOnce(context.Background()) { + t.Fatal("expected the run to be abandoned") + } + + rec, err := health.ReadAbandon(d.cfg.BaseDir) + if err != nil || rec == nil { + t.Fatalf("abandonChild must leave a marker (rec=%+v err=%v)", rec, err) + } + if rec.PID <= 0 { + t.Fatalf("the marker must name the orphan's pid, got %d", rec.PID) + } + if rec.Start == 0 { + t.Fatal("the marker carries no start time: every later check falls back to the cmdline match, and a pid the kernel recycles into another proxsave --backup then pins the alive check DOWN") + } + if cur, ok := procStartTicks(rec.PID); ok && cur != rec.Start { + t.Fatalf("the recorded start time %d is not the orphan's own %d", rec.Start, cur) + } +} + +// TestAStalledOrphanProbeIsNeverReissued guards the resource cost of re-validating a degrade. +// +// probeWithin bounds the WAIT, not the read: the goroutine it gives up on is abandoned, not +// cancelled, and it holds an open /proc file descriptor. Every other caller of that helper is on +// its way out of the process; this one is not -- reviewAbandonDegrade runs it once per heartbeat +// for as long as the degrade stands. Re-issuing a read that has already proved it can block +// would strand a goroutine and an fd on every beat, indefinitely, on the one host an operator is +// actively investigating. +func TestAStalledOrphanProbeIsNeverReissued(t *testing.T) { + d := newTestDaemon(t, nil, nil, time.Hour) + release := make(chan struct{}) + defer close(release) + var reads atomic.Int32 + d.procIdentityIO = func(int, uint64) bool { + reads.Add(1) + <-release // a /proc read the kernel never lets go of + return true + } + + for i := 0; i < 4; i++ { + if !d.pidIsAbandonedChild(4242, 7) { + t.Fatal("a probe that cannot answer must count as the orphan still being there") + } + } + if n := reads.Load(); n != 1 { + t.Fatalf("%d /proc reads are parked in the kernel, one per call; at one call per heartbeat that is a goroutine and an fd stranded every beat, for the life of the daemon", n) + } +} + +// TestSkippedStandaloneBackupKeepsTheDegrade is the other half, and mirrors runOnce's rule +// exactly: ExitBackupSkipped means no backup was performed -- the orphan still holds the lock, +// or backups are off -- so it proves nothing and lifts nothing. +func TestSkippedStandaloneBackupKeepsTheDegrade(t *testing.T) { + first := &fakeReporter{alive: true, backupURL: true} + d := newAbandonDaemon(t, first, 150*time.Millisecond) + if !d.runOnce(context.Background()) { + t.Fatal("expected the run to be abandoned") + } + + restarted := restartedDaemon(t, d, &fakeReporter{alive: true, backupURL: true}) + restarted.cfg.HealthcheckEnabled = true + // As in the supervised half: the orphan is gone, so every other gate on the lift now agrees, + // and only the skip rule can keep the degrade standing. + restarted.pidAliveOverride = func(int) bool { return false } + if err := health.WriteManualOutcome(restarted.cfg.BaseDir, "rid-skip", time.Now().Unix(), types.ExitBackupSkipped.Int()); err != nil { + t.Fatalf("WriteManualOutcome: %v", err) + } + restarted.processManualOutcome(context.Background()) + + if !restarted.aliveDegraded.Load() { + t.Fatal("a skipped standalone run performed no backup; the degrade must stand") + } +} + +// TestDisabledBackupsDoNotInheritTheDegrade closes the trap the degrade would otherwise become. +// The ERROR this path logs tells the operator backups cannot run until the host is cleared, and +// the standard reaction is to turn them off. With BACKUP_ENABLED=false nothing can ever clear +// the marker again -- runOnce returns at its guard before the clear, and a standalone backup +// refuses with ExitBackupSkipped -- so believing the marker here pins the check that pages +// people DOWN forever on a daemon that is perfectly healthy. The backup check is already down +// on its own merits (no run pings anything), which is the honest half of the report. +func TestDisabledBackupsDoNotInheritTheDegrade(t *testing.T) { + first := &fakeReporter{alive: true, backupURL: true} + d := newAbandonDaemon(t, first, 150*time.Millisecond) + if !d.runOnce(context.Background()) { + t.Fatal("expected the run to be abandoned") + } + + rep := &fakeReporter{alive: true, backupURL: true} + off := newTestDaemon(t, rep, nil, time.Hour) + off.cfg.BaseDir = d.cfg.BaseDir + off.cfg.HealthcheckMode = "self" + off.cfg.BackupEnabled = false + off.pidAliveOverride = func(int) bool { return true } // the orphan is still there + off.loadAbandonMarker() + + if off.aliveDegraded.Load() { + t.Fatal("with backups administratively off the alive check must not be pinned DOWN by a degrade nothing can ever lift") + } + off.beat(context.Background()) + if s := rep.snapshot(); s.beats != 1 || s.aliveDown != 0 { + t.Fatalf("a healthy daemon with backups off must beat normally, got beats=%d aliveDown=%d", s.beats, s.aliveDown) + } + // The marker is KEPT: re-enabling backups makes it relevant again, and only a completed run + // (or a vanished orphan) should retire it. + if rec, err := health.ReadAbandon(off.cfg.BaseDir); err != nil || rec == nil { + t.Fatalf("the marker must survive so re-enabling backups restores the degrade (rec=%+v err=%v)", rec, err) + } + back := restartedDaemon(t, d, &fakeReporter{alive: true, backupURL: true}) + if !back.aliveDegraded.Load() { + t.Fatal("re-enabling backups with the orphan still wedged must restore the degrade") + } +} + +// TestVanishedOrphanDoesNotInheritTheDegrade bounds the degrade by the condition it describes +// instead of by a clock. The marker's claim is "pid N is unreapable and holds the backup lock". +// A reboot -- the one action that reliably clears a D-state task -- makes that claim false while +// leaving the file behind, and the marker lives in the identity dir, not /run. Re-validating it +// can only ever CLEAR (a pid that does not exist cannot be our child), never invent, a degrade. +// This exercises the real kill(2) probe, not the test seam. +func TestVanishedOrphanDoesNotInheritTheDegrade(t *testing.T) { + base := t.TempDir() + reaped := exec.Command("/bin/sh", "-c", "exit 0") + if err := reaped.Run(); err != nil { + t.Fatalf("run the throwaway child: %v", err) + } + dead := reaped.Process.Pid // exited AND waited on: this pid is gone + + if err := health.WriteAbandon(base, health.AbandonRecord{PID: dead, RID: "rid-old", TS: time.Now().Unix()}); err != nil { + t.Fatalf("WriteAbandon: %v", err) + } + rep := &fakeReporter{alive: true, backupURL: true} + d := newTestDaemon(t, rep, nil, time.Hour) + d.cfg.BaseDir = base + d.cfg.HealthcheckMode = "self" + d.loadAbandonMarker() + + if d.aliveDegraded.Load() { + t.Fatal("the abandoned child is gone; keeping the alive check DOWN is a false RED on a healed host") + } + if rec, err := health.ReadAbandon(base); err != nil || rec != nil { + t.Fatalf("a marker whose orphan is gone must be retired (rec=%+v err=%v)", rec, err) + } + d.beat(context.Background()) + if s := rep.snapshot(); s.beats != 1 || s.aliveDown != 0 { + t.Fatalf("want a plain success beat once the orphan is gone, got beats=%d aliveDown=%d", s.beats, s.aliveDown) + } +} + +// TestBeatLiftsTheDegradeWhenTheOrphanDies covers the host that heals WITHOUT a restart: the +// NFS server comes back, the task finally leaves D state and dies. Nothing else on the daemon +// would notice until the next scheduled run, up to a whole scheduling period later, and until +// then the alive check is red on a host that is fine -- the mirror image of the false green the +// degrade exists to remove. +func TestBeatLiftsTheDegradeWhenTheOrphanDies(t *testing.T) { + first := &fakeReporter{alive: true, backupURL: true} + d := newAbandonDaemon(t, first, 150*time.Millisecond) + if !d.runOnce(context.Background()) { + t.Fatal("expected the run to be abandoned") + } + + rep := &fakeReporter{alive: true, backupURL: true} + restarted := restartedDaemon(t, d, rep) + restarted.beat(context.Background()) + if s := rep.snapshot(); s.aliveDown != 1 { + t.Fatalf("while the orphan is wedged the beat must report DOWN, got aliveDown=%d", s.aliveDown) + } + + restarted.pidAliveOverride = func(int) bool { return false } // the orphan finally died + restarted.beat(context.Background()) + + if restarted.aliveDegraded.Load() { + t.Fatal("the degrade outlived the orphan it names") + } + if s := rep.snapshot(); s.beats != 1 || s.aliveDown != 1 { + t.Fatalf("the next beat must be a plain success ping, got beats=%d aliveDown=%d", s.beats, s.aliveDown) + } + if rec, err := health.ReadAbandon(restarted.cfg.BaseDir); err != nil || rec != nil { + t.Fatalf("the marker must be retired with the degrade (rec=%+v err=%v)", rec, err) + } +} + +// TestUnreadableMarkerIsStillRetiredByACompletedRun pins the cleanup a degrade-gated clear +// used to skip. A marker this process could not READ still degrades nothing -- we refuse to +// guess -- but it is still a file, and leaving it there means the next process that can read it +// resurrects a degrade for a wedge that ended long ago. +func TestUnreadableMarkerIsStillRetiredByACompletedRun(t *testing.T) { + base := t.TempDir() + // A directory where the marker file belongs: os.ReadFile fails with EISDIR, which is + // ReadAbandon's genuine-read-fault branch rather than its tolerated corrupt-contents one. + if err := os.MkdirAll(health.AbandonPath(base), 0o750); err != nil { + t.Fatalf("stage the unreadable marker: %v", err) + } + d := newTestDaemon(t, &fakeReporter{alive: true, backupURL: true}, shCmd("exit 0"), time.Hour) + d.cfg.BaseDir = base + d.cfg.HealthcheckMode = "self" + d.loadAbandonMarker() + + if d.aliveDegraded.Load() { + t.Fatal("an unreadable marker is not evidence of an abandon; the daemon must not degrade on a guess") + } + d.runOnce(context.Background()) + + if _, err := os.Stat(health.AbandonPath(base)); !os.IsNotExist(err) { + t.Fatalf("a completed run must retire the marker it could not read (stat err=%v)", err) + } +} + +// TestAbandonReportsEvenWhenTheHeartbeatInterlockIsStuck is the regression test for the +// critical defect a naive interlock reintroduces: aliveMu sits on the abandon path, so any +// holder that never lets go would wedge runOnce again -- the same failure this whole change +// removes, one layer down. The ordering the lock buys is a nicety; runOnce returning is the +// invariant. Both pings must still go out. +func TestAbandonReportsEvenWhenTheHeartbeatInterlockIsStuck(t *testing.T) { + rep := &fakeReporter{alive: true, backupURL: true} + d := newAbandonDaemon(t, rep, 150*time.Millisecond) + + // The production wait is 15s of pure wall clock in a package suite; what this test pins is + // that the abandon gives up on the lock AT ALL, which the seam preserves exactly. + d.aliveInterlockWaitOverride = 200 * time.Millisecond + + d.aliveMu.Lock() // a beat parked forever, e.g. inside the cross-process relay-secret flock + defer d.aliveMu.Unlock() + + done := make(chan bool, 1) + go func() { done <- d.runOnce(context.Background()) }() + select { + case abandoned := <-done: + if !abandoned { + t.Fatal("expected the run to be abandoned") + } + case <-time.After(d.aliveInterlockWait() + 20*time.Second): + t.Fatal("runOnce never returned: the alive interlock can still wedge the scheduler") + } + if s := rep.snapshot(); s.hung != 1 || s.aliveDown != 1 { + t.Fatalf("both checks must still be driven DOWN, got hung=%d aliveDown=%d", s.hung, s.aliveDown) + } +} + +// TestAbandonPingsTheBackupCheckBeforeTouchingTheAliveInterlock pins the ordering rule that +// keeps the primary signal free of an unrelated lock: the backup /fail -- the report this +// whole change exists to make reachable -- must be on the wire before aliveMu is contended +// for, so a stuck heartbeat can never delay it. +func TestAbandonPingsTheBackupCheckBeforeTouchingTheAliveInterlock(t *testing.T) { + rep := &fakeReporter{alive: true, backupURL: true} + d := newAbandonDaemon(t, rep, 150*time.Millisecond) + + d.aliveMu.Lock() + done := make(chan bool, 1) + go func() { done <- d.runOnce(context.Background()) }() + + reported := false + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) && !reported { + reported = rep.snapshot().hung == 1 + time.Sleep(10 * time.Millisecond) + } + d.aliveMu.Unlock() + <-done // join before the temp BaseDir is torn down + + if !reported { + t.Fatal("the backup hang /fail was gated on the alive interlock; it must be sent before it") + } +} + +// TestScheduleLoopUnwindsOnAbandon proves the wedge is gone at the level the report described: +// runOnce is called synchronously from the loop, so today the scheduler never comes back and +// no further backup is ever scheduled. The clock is frozen just before the scheduled time so +// the loop arms one short timer; it never reaches a second iteration, so the frozen clock +// cannot spin. +func TestScheduleLoopUnwindsOnAbandon(t *testing.T) { + rep := &fakeReporter{alive: true, backupURL: true} + d := newAbandonDaemon(t, rep, 150*time.Millisecond) + d.cfg.SchedulerTime = "03:00" + d.now = func() time.Time { return time.Date(2026, 1, 2, 2, 59, 59, 750*int(time.Millisecond), time.UTC) } + + done := make(chan bool, 1) + go func() { done <- d.scheduleLoop(context.Background()) }() + select { + case abandoned := <-done: + if !abandoned { + t.Fatal("scheduleLoop must report the abandoned run up to run()") + } + case <-time.After(10 * time.Second): + t.Fatal("scheduleLoop never returned: the scheduler is still wedged behind the child") + } +} + +// TestRunExitsAfterAbandoningUnreapableChild pins the whole unwind, and is the regression test +// for the deadlock a partial fix would introduce: scheduleLoop propagates the abandon, run() +// cancels its OWN derived context so the SIGUSR1 waker (started unconditionally and joined by +// wg.Wait()) actually returns, and the daemon exits non-zero so systemd restarts it. Without +// that derived cancel the caller's context is still live and wg.Wait() blocks forever -- one +// hang traded for another. +// +// HealthcheckEnabled stays false, so no heartbeat/update loop starts and nothing touches the +// network; the waker alone is enough to deadlock wg.Wait(). +func TestRunExitsAfterAbandoningUnreapableChild(t *testing.T) { + rep := &fakeReporter{alive: true, backupURL: true} + d := newAbandonDaemon(t, rep, 150*time.Millisecond) + d.cfg.SchedulerTime = "03:00" + d.now = func() time.Time { return time.Date(2026, 1, 2, 2, 59, 59, 750*int(time.Millisecond), time.UTC) } + + code := make(chan int, 1) + go func() { code <- d.run(context.Background()) }() + select { + case got := <-code: + if got != types.ExitBackupError.Int() { + t.Fatalf("run() exit = %d, want %d (abandoning is not a clean stop, and exit 1 is documented as benign)", + got, types.ExitBackupError.Int()) + } + case <-time.After(20 * time.Second): + t.Fatal("run() did not return after abandoning the child: the daemon can never be restarted") + } +} + +// TestReapWaitCoversTheKillGrace pins the production ordering the overridden tests cannot +// exercise: the daemon must never declare a child unreapable before os/exec has actually sent +// the SIGKILL. Anything less would abandon children that a kill would still have collected. +func TestReapWaitCoversTheKillGrace(t *testing.T) { + d := &daemon{} + if d.killGrace() != daemonKillGrace { + t.Fatalf("killGrace() = %s, want the production %s", d.killGrace(), daemonKillGrace) + } + if d.reapWait() <= d.killGrace() { + t.Fatalf("reapWait() = %s must exceed the kill grace %s, or a child is abandoned before the SIGKILL is even sent", + d.reapWait(), d.killGrace()) + } + if d.reapWait() != daemonKillGrace+daemonReapSlack { + t.Fatalf("reapWait() = %s, want killGrace+slack = %s", d.reapWait(), daemonKillGrace+daemonReapSlack) + } +} + +// corruptMarker stages the case the health package deliberately tolerates: a marker whose +// PRESENCE is the signal but whose contents are unreadable, so ReadAbandon yields the zero +// record (pid 0, no rid). WriteAbandon does not fsync and these hosts are typically hard-reset +// by the operator, so a truncated file is the ordinary way this happens -- not a curiosity. +func corruptMarker(t *testing.T, base string) { + t.Helper() + if err := health.WriteAbandon(base, health.AbandonRecord{PID: 1}); err != nil { + t.Fatalf("WriteAbandon: %v", err) + } + if err := os.WriteFile(health.AbandonPath(base), []byte("{not json"), 0o600); err != nil { + t.Fatalf("corrupt the marker: %v", err) + } +} + +// TestARecycledPidDoesNotPinTheDegradeForever is the mirror of +// TestCompletedRunOverALiveOrphanKeepsTheDegrade, and covers the false RED that gating every +// lift path on a bare liveness probe creates. +// +// "Is pid N alive" does not identify a PROCESS: the kernel recycles pid numbers within a boot. +// The mount heals, the orphan finally dies while nothing is watching -- the daemon was stopped +// for the repair, or healthchecks are off so no beat ever reviews the marker -- and the number +// is handed to the next long-lived process the host starts. From then on the probe answers +// "still there" forever: startup re-degrades, the beat never lifts, and a completed run does +// not lift either, so the check that pages people is RED on a host whose backups are running +// perfectly, until somebody deletes the marker by hand. The boot-generation check cannot help +// -- the reuse happened WITHIN this boot. +// +// This runs the real probe, not the test seam, and names a pid that is unquestionably alive +// and unquestionably not a backup child: the test process itself. +func TestARecycledPidDoesNotPinTheDegradeForever(t *testing.T) { + cases := []struct { + name string + start uint64 + }{ + // A tick count no process on this host can have (~348 years of uptime at 100 Hz), so + // the exact identity test must reject it. + {"start time recorded", 1 << 40}, + // A marker written before that field existed: the cmdline fallback must reject it too. + {"legacy marker with no start time", 0}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + base := t.TempDir() + if err := health.WriteAbandon(base, health.AbandonRecord{ + PID: os.Getpid(), Start: tc.start, RID: "rid-old", TS: time.Now().Unix(), + }); err != nil { + t.Fatalf("WriteAbandon: %v", err) + } + rep := &fakeReporter{alive: true, backupURL: true} + d := newTestDaemon(t, rep, nil, time.Hour) + d.cfg.BaseDir = base + d.cfg.HealthcheckMode = "self" + d.loadAbandonMarker() + + if d.aliveDegraded.Load() { + t.Fatal("the pid was recycled by an unrelated process; keeping the alive check DOWN is a false RED nothing can ever lift") + } + if rec, err := health.ReadAbandon(base); err != nil || rec != nil { + t.Fatalf("a marker whose pid is no longer our child must be retired (rec=%+v err=%v)", rec, err) + } + d.beat(context.Background()) + if s := rep.snapshot(); s.beats != 1 || s.aliveDown != 0 { + t.Fatalf("want a plain success beat over a recycled pid, got beats=%d aliveDown=%d", s.beats, s.aliveDown) + } + }) + } +} + +// TestACorruptMarkerIsNotLiftedByAFailedRun closes the false GREEN the pid-based carve-out left +// open on exactly the host that is hardest to reason about. +// +// A corrupt marker degrades (presence is the signal), but it names no pid, so the orphan probe +// has nothing to ask. Treating that as "there is no orphan to outlive" hands the lift to the +// exit code alone -- and the exit code is what the whole clearAbandonMarkerOnCompletedRun +// contract says proves nothing, because the backup lock is checked LAST, after the very gates a +// dead mount fails. The operator's `proxsave --backup`, run to see what is wrong, dies on the +// disk-space check and re-greens the alive check over an orphan that has not moved. +// +// A run that actually SUCCEEDED is different in kind: it passed the lock check, so a backup +// completed end to end and backups are demonstrably not dead. That is the escape, so the +// degrade is still bounded rather than permanent. +func TestACorruptMarkerIsNotLiftedByAFailedRun(t *testing.T) { + base := t.TempDir() + corruptMarker(t, base) + + d := newTestDaemon(t, &fakeReporter{alive: true, backupURL: true}, + shCmd("exit "+strconv.Itoa(types.ExitBackupError.Int())), time.Hour) + d.cfg.BaseDir = base + d.cfg.HealthcheckMode = "self" + d.loadAbandonMarker() + if !d.aliveDegraded.Load() { + t.Fatal("a marker whose contents are unreadable is still an abandon; it must degrade") + } + + d.runOnce(context.Background()) + if !d.aliveDegraded.Load() { + t.Fatal("a run that failed before the lock check proves nothing; the degrade must stand even when the marker names no pid") + } + if rec, err := health.ReadAbandon(base); err != nil || rec == nil { + t.Fatalf("the marker must survive so a later restart still inherits the degrade (rec=%+v err=%v)", rec, err) + } + + // The escape: a backup that actually succeeded took the lock the orphan would be holding. + d.newBackupCmd = shCmd("exit 0") + d.runOnce(context.Background()) + if d.aliveDegraded.Load() { + t.Fatal("a successful backup passed the lock check; nothing is left for the degrade to mean") + } + if rec, err := health.ReadAbandon(base); err != nil || rec != nil { + t.Fatalf("the marker must be retired with the degrade (rec=%+v err=%v)", rec, err) + } +} + +// TestACorruptMarkerIsLiftedByARunThatOnlyWarned closes the mirror defect of the test above: a +// service-alive DOWN on a healthy host that NOTHING can lift. +// +// A corrupt marker names no pid, so the probe has nothing to ask and every probe-based lift path +// is closed; it carries no timestamp either, so the boot-generation check cannot date it and a +// reboot does not retire it. The run's own exit code is therefore the only escape there is -- +// and exit 1 is not a failure. applyIssueExitCode promotes a CLEAN run to it when the run logged +// warnings (a run with real errors becomes 4 instead), and docs/TROUBLESHOOTING.md documents a +// routine state in which every run on a perfectly healthy host exits 1: unacknowledged release +// notes after an upgrade. Refusing it there leaves the check that pages people RED until somebody +// deletes the file by hand. +func TestACorruptMarkerIsLiftedByARunThatOnlyWarned(t *testing.T) { + base := t.TempDir() + corruptMarker(t, base) + + // First: a reboot really is no escape for this marker, so the exit code has to be one. + rebooted := newTestDaemon(t, &fakeReporter{alive: true, backupURL: true}, nil, time.Hour) + rebooted.cfg.BaseDir = base + rebooted.cfg.HealthcheckMode = "self" + rebooted.bootUnixOverride = func() int64 { return time.Now().Add(time.Hour).Unix() } + rebooted.loadAbandonMarker() + if !rebooted.aliveDegraded.Load() { + t.Fatal("a marker with no pid and no timestamp cannot be dated; the boot check must not pretend otherwise") + } + + d := newTestDaemon(t, &fakeReporter{alive: true, backupURL: true}, + shCmd("exit "+strconv.Itoa(types.ExitGenericError.Int())), time.Hour) + d.cfg.BaseDir = base + d.cfg.HealthcheckMode = "self" + d.loadAbandonMarker() + if !d.aliveDegraded.Load() { + t.Fatal("a marker whose contents are unreadable is still an abandon; it must degrade") + } + + d.runOnce(context.Background()) + if d.aliveDegraded.Load() { + t.Fatal("a backup that finished with warnings passed the lock check; keeping the alive check DOWN is a false RED no run, no restart and no reboot can lift") + } + if rec, err := health.ReadAbandon(base); err != nil || rec != nil { + t.Fatalf("the marker must be retired with the degrade (rec=%+v err=%v)", rec, err) + } +} + +// TestAKeptMarkerSurvivesAStandaloneRunThatFailedBeforeTheLock covers the quieter half of the +// same rule: the clear paths that run when THIS process never degraded at all. +// +// With BACKUP_ENABLED=false the marker is deliberately kept for a later re-enable, and nothing +// is degraded. A `proxsave --backup` that dies on a pre-flight gate is exactly the evidence +// clearAbandonMarkerOnCompletedRun exists to disbelieve, and deleting the marker on it costs +// nothing visible in this process -- the whole price lands on the successor, which comes up +// fully green over an orphan that still holds the backup lock while every scheduled run exits +// ExitBackupSkipped and pings nothing. +func TestAKeptMarkerSurvivesAStandaloneRunThatFailedBeforeTheLock(t *testing.T) { + first := &fakeReporter{alive: true, backupURL: true} + d := newAbandonDaemon(t, first, 150*time.Millisecond) + if !d.runOnce(context.Background()) { + t.Fatal("expected the run to be abandoned") + } + + off := newTestDaemon(t, &fakeReporter{alive: true, backupURL: true}, nil, time.Hour) + off.cfg.BaseDir = d.cfg.BaseDir + off.cfg.HealthcheckMode = "self" + off.cfg.HealthcheckEnabled = true + off.cfg.BackupEnabled = false + off.pidAliveOverride = func(int) bool { return true } // the orphan is still wedged + off.loadAbandonMarker() + if off.aliveDegraded.Load() { + t.Fatal("with backups administratively off nothing may be degraded") + } + + // The operator put BACKUP_ENABLED back in the config and proved it by hand before + // restarting the daemon, and the run died on the disk-space gate -- long before the lock. + if err := health.WriteManualOutcome(off.cfg.BaseDir, "rid-early-fail", time.Now().Unix(), types.ExitBackupError.Int()); err != nil { + t.Fatalf("WriteManualOutcome: %v", err) + } + off.processManualOutcome(context.Background()) + + if rec, err := health.ReadAbandon(off.cfg.BaseDir); err != nil || rec == nil { + t.Fatalf("a run that failed before the lock check deleted a marker that was kept on purpose (rec=%+v err=%v)", rec, err) + } + back := restartedDaemon(t, d, &fakeReporter{alive: true, backupURL: true}) + if !back.aliveDegraded.Load() { + t.Fatal("the successor inherited nothing and comes up green over an orphan that still holds the backup lock") + } +} + +// TestAnUnreadableMarkerSurvivesARunThatFailedBeforeTheLock is the other un-degraded state, and +// the more likely of the two: ReadAbandon's genuine-read-fault branch is reached on precisely +// the host whose BaseDir I/O is wedged -- the same fault that parks a backup child in D state. +// This process never even saw the record, so it knows nothing about the orphan it names, and a +// pre-lock failure tells it nothing either. The cleanup it does owe the identity dir is real, +// but it waits for evidence. +func TestAnUnreadableMarkerSurvivesARunThatFailedBeforeTheLock(t *testing.T) { + base := t.TempDir() + // A directory where the marker file belongs: os.ReadFile fails with EISDIR, which is + // ReadAbandon's genuine-read-fault branch rather than its tolerated corrupt-contents one. + if err := os.MkdirAll(health.AbandonPath(base), 0o750); err != nil { + t.Fatalf("stage the unreadable marker: %v", err) + } + d := newTestDaemon(t, &fakeReporter{alive: true, backupURL: true}, + shCmd("exit "+strconv.Itoa(types.ExitBackupError.Int())), time.Hour) + d.cfg.BaseDir = base + d.cfg.HealthcheckMode = "self" + d.loadAbandonMarker() + if d.aliveDegraded.Load() { + t.Fatal("an unreadable marker is not evidence of an abandon; the daemon must not degrade on a guess") + } + + d.runOnce(context.Background()) + if _, err := os.Stat(health.AbandonPath(base)); err != nil { + t.Fatalf("a run that failed before the lock check deleted a marker this daemon never read; the successor comes up green over an orphan that may still hold the lock (stat err=%v)", err) + } + + // ...and the cleanup is not lost: a run that DID reach the lock still retires it. + d.newBackupCmd = shCmd("exit 0") + d.runOnce(context.Background()) + if _, err := os.Stat(health.AbandonPath(base)); !os.IsNotExist(err) { + t.Fatalf("a completed run must still retire the marker it could not read (stat err=%v)", err) + } +} + +// unstartableCmd builds a child that can never be FORKED: the path does not exist, so +// cmd.Start fails and no process is ever created. superviseChild deliberately folds that into +// reaped=true (there is nothing to abandon and no pid to leak), and exitCodeFromErr then +// SYNTHESISES exit code 1 for it -- TestExitCodeFromErr pins that mapping. A fork/exec failure +// is ordinary on the host class this whole path exists for: EAGAIN/ENOMEM on a box already +// accumulating D-state tasks, a BASE_DIR mount that swallowed the binary, or a path that +// vanished under a package upgrade. +func unstartableCmd() func(ctx context.Context) *exec.Cmd { + return func(ctx context.Context) *exec.Cmd { + return exec.CommandContext(ctx, "/nonexistent/proxsave/binary/xyz", "--backup") + } +} + +// TestAChildThatNeverStartedProvesNothingAboutTheLock closes the false GREEN that the exit +// code alone opens on the two branches with no pid to probe. +// +// The daemon-side 1 and the child's own 1 are different facts wearing the same number. +// exitProvesLockWasTaken reasons about the CHILD's exit codes -- there 1 is a clean run that +// only warned, which is why it must qualify (see TestACorruptMarkerIsLiftedByARunThatOnlyWarned) +// -- but exitCodeFromErr also synthesises 1 for an error that carries no wait status at all, +// above all a cmd.Start failure. Nothing was forked, so nothing reached the backup lock, and +// nothing may retire the marker. +// +// Both un-probeable states are covered because the exit code is the ONLY evidence in each: +// the corrupt marker degrades but names no pid, and the unreadable one never degraded and is +// reached on precisely the host whose BaseDir I/O is wedged. Deleting on either hands the +// successor nothing, and it beats green over an orphan that may still hold the lock. +func TestAChildThatNeverStartedProvesNothingAboutTheLock(t *testing.T) { + t.Run("corrupt marker", func(t *testing.T) { + base := t.TempDir() + corruptMarker(t, base) + + d := newTestDaemon(t, &fakeReporter{alive: true, backupURL: true}, unstartableCmd(), time.Hour) + d.cfg.BaseDir = base + d.cfg.HealthcheckMode = "self" + d.loadAbandonMarker() + if !d.aliveDegraded.Load() { + t.Fatal("a marker whose contents are unreadable is still an abandon; it must degrade") + } + + d.runOnce(context.Background()) + if !d.aliveDegraded.Load() { + t.Fatal("no child was ever forked, so nothing reached the backup lock; the degrade must stand") + } + if rec, err := health.ReadAbandon(base); err != nil || rec == nil { + t.Fatalf("a run whose child could not even be started deleted the marker (rec=%+v err=%v)", rec, err) + } + }) + + t.Run("unreadable marker", func(t *testing.T) { + base := t.TempDir() + // ReadAbandon's genuine-read-fault branch: os.ReadFile fails with EISDIR, so this + // process never degrades and only owes the identity dir a cleanup once a run proves + // the wedge is over. + if err := os.MkdirAll(health.AbandonPath(base), 0o750); err != nil { + t.Fatalf("stage the unreadable marker: %v", err) + } + d := newTestDaemon(t, &fakeReporter{alive: true, backupURL: true}, unstartableCmd(), time.Hour) + d.cfg.BaseDir = base + d.cfg.HealthcheckMode = "self" + d.loadAbandonMarker() + if d.aliveDegraded.Load() { + t.Fatal("an unreadable marker is not evidence of an abandon; the daemon must not degrade on a guess") + } + + d.runOnce(context.Background()) + if _, err := os.Stat(health.AbandonPath(base)); err != nil { + t.Fatalf("a run whose child could not even be started retired a marker this daemon never read; the successor comes up green over an orphan that may still hold the lock (stat err=%v)", err) + } + + // ...and the cleanup is not lost: a child that really RAN and only warned still + // retires it, so this is not a blanket refusal of exit 1. + d.newBackupCmd = shCmd("exit " + strconv.Itoa(types.ExitGenericError.Int())) + d.runOnce(context.Background()) + if _, err := os.Stat(health.AbandonPath(base)); !os.IsNotExist(err) { + t.Fatalf("a run that really reached the lock must still retire the marker (stat err=%v)", err) + } + }) +} + +// TestAFailedMarkerRemovalIsRetriedByALaterRun pins the gate that decides whether anything ever +// touches the file again. abandonMarkerOnDisk is dropped before the unlink is known to have +// happened, and the unlink's error is only Debug-logged, so a removal that FAILS leaves a marker +// no later caller in this process will retry -- and the next daemon reads it and degrades for a +// wedge that ended long ago. +func TestAFailedMarkerRemovalIsRetriedByALaterRun(t *testing.T) { + base := t.TempDir() + // A marker path that cannot be unlinked: a NON-EMPTY directory where the file belongs. + // os.ReadFile fails with EISDIR and os.Remove with ENOTEMPTY, which is the shape of any + // removal fault on an identity dir the daemon cannot fully write. + if err := os.MkdirAll(health.AbandonPath(base), 0o750); err != nil { + t.Fatalf("stage the marker: %v", err) + } + blocker := filepath.Join(health.AbandonPath(base), "blocker") + if err := os.WriteFile(blocker, []byte("x"), 0o600); err != nil { + t.Fatalf("stage the blocker: %v", err) + } + + d := newTestDaemon(t, &fakeReporter{alive: true, backupURL: true}, shCmd("exit 0"), time.Hour) + d.cfg.BaseDir = base + d.cfg.HealthcheckMode = "self" + d.loadAbandonMarker() + + d.runOnce(context.Background()) // the removal is attempted and fails + if _, err := os.Stat(health.AbandonPath(base)); err != nil { + t.Fatalf("the removal was supposed to fail, leaving the path in place: %v", err) + } + + // The identity dir recovers, and the next completed run must try again. + if err := os.Remove(blocker); err != nil { + t.Fatalf("unblock the removal: %v", err) + } + d.runOnce(context.Background()) + if _, err := os.Stat(health.AbandonPath(base)); !os.IsNotExist(err) { + t.Fatalf("a removal that failed closed the gate for good, so no later run retries it and the marker survives to re-degrade the next daemon (stat err=%v)", err) + } +} + +// TestAStragglingMarkerRemovalCannotEatAFreshAbandonMarker covers the one interleaving abandonMu +// cannot close, on the one host class this path exists for. +// +// clearAbandonMarker unlinks inside runWithin, and runWithin gives up WAITING without cancelling +// anything -- nothing in userspace can recall a syscall the kernel is holding. On a wedged +// BaseDir the lock is therefore released with the removal still queued: abandonChild then takes +// it, latches, and writes the marker for a NEW orphan, and the straggler finally lands on top of +// that fresh file. The successor daemon inherits nothing and beats green over a live orphan -- +// the exact outcome the barrier was chosen to prevent, reached silently. +func TestAStragglingMarkerRemovalCannotEatAFreshAbandonMarker(t *testing.T) { + base := t.TempDir() + // The precondition: this process inherited a marker, so its clear paths are armed. + corruptMarker(t, base) + + rep := &fakeReporter{alive: true, backupURL: true} + d := newAbandonDaemon(t, rep, 150*time.Millisecond) + d.cfg.BaseDir = base + d.cfg.HealthcheckMode = "self" + d.aliveInterlockWaitOverride = 100 * time.Millisecond + + entered, release, unlinked := make(chan struct{}), make(chan struct{}), make(chan struct{}) + d.clearAbandonMarkerIO = func() error { + close(entered) + <-release // an unlink parked in the kernel, long past daemonAbandonIOWait + err := health.ClearAbandon(base) + close(unlinked) + return err + } + d.loadAbandonMarker() + + clearDone := make(chan struct{}) + go func() { + defer close(clearDone) + d.clearAbandonMarkerOnCompletedRun("a supervised backup completed", 0) + }() + <-entered + + // ...and while that removal is stuck, the daemon wedges on a NEW child and abandons it. + if !d.runOnce(context.Background()) { + t.Fatal("expected the run to be abandoned") + } + <-clearDone + fresh, err := health.ReadAbandon(base) + if err != nil || fresh == nil || fresh.PID <= 0 { + t.Fatalf("abandonChild must leave a marker naming the new orphan (rec=%+v err=%v)", fresh, err) + } + + close(release) // the parked unlink finally lands, on top of the fresh marker + <-unlinked + + deadline := time.Now().Add(5 * time.Second) + var rec *health.AbandonRecord + for time.Now().Before(deadline) { + if rec, err = health.ReadAbandon(base); err == nil && rec != nil && rec.PID == fresh.PID { + break + } + time.Sleep(10 * time.Millisecond) + } + if rec == nil || rec.PID != fresh.PID { + t.Fatalf("a removal that outlived the barrier deleted the marker written for orphan pid=%d (rec=%+v); the next daemon comes up green over a live orphan", fresh.PID, rec) + } +} + +// TestAFreshAbandonMarkerSurvivesAStandaloneHandoff pins the barrier that keeps the marker +// written for THIS process's orphan out of reach of the clear paths reasoning about the +// INHERITED one. +// +// The two are the same file, so whoever writes last wins. abandonChild persists its marker and +// then spends up to the whole alive interlock wait plus a ping before returning, and the SIGUSR1 +// waker goroutine stays live for all of it (run() only stops the loops after scheduleLoop +// returns). A `proxsave --backup` handing off in that window -- precisely what an operator runs +// when they notice the wedge -- takes processManualOutcome straight to the clear, and the +// successor comes up fully green over a live orphan that still holds the backup lock. Guarding +// one caller does not fix this; the barrier belongs where the file is touched. +func TestAFreshAbandonMarkerSurvivesAStandaloneHandoff(t *testing.T) { + base := t.TempDir() + // The precondition: this process inherited a marker, so its clear paths are armed. Corrupt + // is the easiest way there and also the nastiest -- it names no pid to re-check. + corruptMarker(t, base) + + rep := &fakeReporter{alive: true, backupURL: true} + d := newAbandonDaemon(t, rep, 150*time.Millisecond) + d.cfg.BaseDir = base + d.cfg.HealthcheckMode = "self" + d.cfg.HealthcheckEnabled = true + d.aliveInterlockWaitOverride = 200 * time.Millisecond + d.loadAbandonMarker() + + // ...and then it wedges on a NEW child and abandons it. + if !d.runOnce(context.Background()) { + t.Fatal("expected the run to be abandoned") + } + fresh, err := health.ReadAbandon(base) + if err != nil || fresh == nil || fresh.PID <= 0 { + t.Fatalf("abandonChild must leave a marker naming the new orphan (rec=%+v err=%v)", fresh, err) + } + + // The handoff lands while the abandon is still unwinding. + if err := health.WriteManualOutcome(base, "rid-manual", time.Now().Unix(), 0); err != nil { + t.Fatalf("WriteManualOutcome: %v", err) + } + d.processManualOutcome(context.Background()) + + rec, err := health.ReadAbandon(base) + if err != nil { + t.Fatalf("ReadAbandon: %v", err) + } + if rec == nil || rec.PID != fresh.PID { + t.Fatalf("the marker written for orphan pid=%d was deleted (rec=%+v); the next daemon comes up green over a live orphan", fresh.PID, rec) + } +} + +// TestARetiredMarkerLeavesNoPidBehindToProbe covers the operator-facing half of the +// boot-generation discard. Retiring the marker is only half the job: a pid left behind in the +// daemon's fields is probed by every completed run for the rest of the process's life, and +// after a reboot that number belongs to something else, so each successful backup announces +// that the service-alive check is being held DOWN -- while it is green. The healed host is +// exactly the host the discard exists to serve. +func TestARetiredMarkerLeavesNoPidBehindToProbe(t *testing.T) { + first := &fakeReporter{alive: true, backupURL: true} + d := newAbandonDaemon(t, first, 150*time.Millisecond) + if !d.runOnce(context.Background()) { + t.Fatal("expected the run to be abandoned") + } + + rebooted := newTestDaemon(t, &fakeReporter{alive: true, backupURL: true}, shCmd("exit 0"), time.Hour) + rebooted.cfg.BaseDir = d.cfg.BaseDir + rebooted.cfg.HealthcheckMode = "self" + rebooted.pidAliveOverride = func(int) bool { return false } // the reboot freed the orphan + rebooted.bootUnixOverride = func() int64 { return time.Now().Add(time.Hour).Unix() } + rebooted.loadAbandonMarker() + + if rebooted.aliveDegraded.Load() { + t.Fatal("a marker written before the current boot must not degrade") + } + if rebooted.abandonPID != 0 { + t.Fatalf("the retired marker left pid=%d behind; every completed run now probes an unrelated process and reports it is holding DOWN a check that is green", rebooted.abandonPID) + } + rebooted.runOnce(context.Background()) + if rebooted.aliveDegraded.Load() { + t.Fatal("nothing may re-raise a degrade the daemon retired at startup") + } +} + +// TestAbandonReportsThroughAReporterInstalledMidRun pins the re-read the two MANDATED signals +// depend on. runOnce captures the reporter once, at the top of a run that may last +// MAX_RUN_DURATION; in centralized mode a daemon that started unpaired resolves its URLs on a +// later beat and installs one behind the run's back. Without re-reading it, the daemon that +// most needs to report -- one whose monitor only just became reachable -- drops BOTH the backup +// hang and the alive degrade and exits silently. +func TestAbandonReportsThroughAReporterInstalledMidRun(t *testing.T) { + late := &fakeReporter{alive: true, backupURL: true} + d := newAbandonDaemon(t, nil, 150*time.Millisecond) // no URLs resolved when the run starts + child := sigtermProofCmd("3") + d.newBackupCmd = func(ctx context.Context) *exec.Cmd { + d.setReporter(late) // the lazy centralized re-resolve, landing mid-run + return child(ctx) + } + + if !d.runOnce(context.Background()) { + t.Fatal("expected the run to be abandoned") + } + if s := late.snapshot(); s.hung != 1 || s.aliveDown != 1 { + t.Fatalf("the abandon must report through the reporter resolved during the run, got hung=%d aliveDown=%d", s.hung, s.aliveDown) + } +} + +// TestDaemonFileCleanupCannotHoldTheExit guards the last five lines of the abandon path. The +// daemon abandons an unreapable child in order to EXIT and be restarted by systemd, and +// everything on the way out is deadline-bounded for one reason: BaseDir may be on the very +// filesystem that parked the child in D state. run()'s final defer removes the pid file and +// .daemon_info.json from that same directory, so an unbounded unlink there strands the process +// after all the bounded work -- alive, unable to die, and never restarted. +func TestDaemonFileCleanupCannotHoldTheExit(t *testing.T) { + d := newTestDaemon(t, nil, nil, time.Hour) + release := make(chan struct{}) + defer close(release) + d.removeDaemonFilesIO = func() { <-release } // an unlink that never returns + + done := make(chan struct{}) + go func() { defer close(done); d.removeDaemonFiles() }() + select { + case <-done: + case <-time.After(daemonAbandonIOWait + 10*time.Second): + t.Fatal("the exit is hostage to the identity dir: on a wedged BaseDir the daemon never returns from run() and systemd never restarts it") + } +} diff --git a/cmd/proxsave/daemon_service.go b/cmd/proxsave/daemon_service.go index dc3f3ee3..359c8c93 100644 --- a/cmd/proxsave/daemon_service.go +++ b/cmd/proxsave/daemon_service.go @@ -37,6 +37,23 @@ var daemonUnitPath = filepath.Join(daemonUnitDir, daemonUnitName) // supervisor (Restart=always); the daemon schedules internally. A non-empty // configPath is pinned with --config so the unit uses the same backup.env the // install/upgrade wrote. +// +// The kill directives are deliberately LEFT AT THEIR DEFAULTS (KillMode=control-group, +// TimeoutStopSec=DefaultTimeoutStopSec). It is tempting to set KillMode=process so that a +// backup child the daemon had to abandon (abandonChild in daemon.go) -- parked in +// TASK_UNINTERRUPTIBLE, still inside this unit's cgroup, never going to leave it -- cannot +// hold the stop job through its timeout phases and delay Restart=always. That trade is a bad +// one. The cgroup-wide kill is the ONLY thing that collects the abandoned child's own +// descendants: the daemon signals a single pid (cmd.Cancel -> cmd.Process.Signal, and +// os/exec's WaitDelay escalation is likewise Process.Kill), nothing here ever creates or +// signals a process group, and a child stuck in D state cannot run its own context-cancel +// unwind to tear down the tar/pigz/rclone/proxmox-backup-client processes it started against +// the same wedged mount. Under KillMode=process those survive the daemon's exit, its restart, +// and every ordinary `systemctl stop`, with nothing left to sweep them; systemd.kill(5) says +// as much ("not recommended ... allows processes to escape the service manager's lifecycle"). +// Leaving the default costs a delayed restart -- minutes, for a scheduler that runs once a +// day -- and buys back descendant cleanup on every stop. Nothing in the unit changed for the +// abandon path, so nothing has to be redelivered to hosts already running an installed unit. func buildDaemonUnit(execToken, configPath string) string { exec := strings.TrimSpace(execToken) if exec == "" { diff --git a/cmd/proxsave/daemon_service_test.go b/cmd/proxsave/daemon_service_test.go index 9f9cfa78..3c0dcfd4 100644 --- a/cmd/proxsave/daemon_service_test.go +++ b/cmd/proxsave/daemon_service_test.go @@ -66,6 +66,23 @@ func TestBuildDaemonUnitWithConfig(t *testing.T) { } } +// TestBuildDaemonUnitKeepsTheCgroupWideKill pins a deliberate NON-change. Trading +// KillMode=control-group for KillMode=process would shorten the restart after the daemon +// abandons an unreapable child, but it is the cgroup-wide kill that collects that child's own +// descendants (tar / pigz / rclone / proxmox-backup-client): the daemon only ever signals the +// single child pid, no process group is ever created, and a child in D state cannot run its +// own cleanup. Weakening it would orphan those on every stop and restart, not just on the +// abandon path, so the unit stays on systemd's defaults and the restart is simply slower. +func TestBuildDaemonUnitKeepsTheCgroupWideKill(t *testing.T) { + u := buildDaemonUnit("/usr/local/bin/proxsave", "") + if strings.Contains(u, "KillMode=") { + t.Errorf("the unit must not weaken systemd's cgroup-wide kill; nothing else cleans up the backup child's descendants:\n%s", u) + } + if !strings.Contains(u, "\nRestart=always\n") || !strings.Contains(u, "\nRestartSec=10\n") { + t.Errorf("the abandon path relies on systemd restarting the daemon:\n%s", u) + } +} + func TestBuildDaemonUnitFallbacks(t *testing.T) { // Empty exec token -> canonical path; empty config -> no --config. u := buildDaemonUnit("", "") diff --git a/cmd/proxsave/daemon_test.go b/cmd/proxsave/daemon_test.go index 981227ec..b019f928 100644 --- a/cmd/proxsave/daemon_test.go +++ b/cmd/proxsave/daemon_test.go @@ -22,6 +22,8 @@ type fakeReporter struct { finished int hung int beats int + aliveDown int // AliveDegraded calls: the /fail that drives the service-alive check DOWN + lastAliveReason string // the BODY of that /fail: what the operator actually reads in the alert lastCode int lastRid string lastTail string @@ -44,6 +46,16 @@ func (f *fakeReporter) Heartbeat(ctx context.Context) error { f.beats++ return nil } +func (f *fakeReporter) AliveDegraded(ctx context.Context, reason string) error { + f.mu.Lock() + defer f.mu.Unlock() + if !f.alive { // no alive URL resolved: mirror health.Reporter's ErrNoAliveURL + return health.ErrNoAliveURL + } + f.aliveDown++ + f.lastAliveReason = reason + return nil +} func (f *fakeReporter) RunStarted(ctx context.Context, rid string) error { f.mu.Lock() defer f.mu.Unlock() @@ -106,7 +118,7 @@ func (f *fakeReporter) Ping(ctx context.Context, name, suffix, rid, body, label func (f *fakeReporter) snapshot() fakeReporter { f.mu.Lock() defer f.mu.Unlock() - return fakeReporter{started: f.started, finished: f.finished, hung: f.hung, beats: f.beats, lastCode: f.lastCode, lastRid: f.lastRid, lastTail: f.lastTail, updates: f.updates, updatesReported: f.updatesReported, lastAvailable: f.lastAvailable, pings: append([]fakePing(nil), f.pings...)} + return fakeReporter{started: f.started, finished: f.finished, hung: f.hung, beats: f.beats, aliveDown: f.aliveDown, lastAliveReason: f.lastAliveReason, lastCode: f.lastCode, lastRid: f.lastRid, lastTail: f.lastTail, updates: f.updates, updatesReported: f.updatesReported, lastAvailable: f.lastAvailable, pings: append([]fakePing(nil), f.pings...)} } func newTestDaemon(t *testing.T, rep backupReporter, cmdFn func(ctx context.Context) *exec.Cmd, maxRun time.Duration) *daemon { diff --git a/internal/health/abandon.go b/internal/health/abandon.go new file mode 100644 index 00000000..7f95ed46 --- /dev/null +++ b/internal/health/abandon.go @@ -0,0 +1,100 @@ +// abandon.go records that the daemon gave up on a backup child the kernel would not let it +// reap (a task parked in TASK_UNINTERRUPTIBLE behind a dead NFS/CIFS mount or a wedged +// device). The daemon EXITS on that path and systemd restarts it seconds later, so the fact +// has to outlive the process: the restarted daemon must keep reporting the service-alive +// check DOWN instead of sending a success heartbeat that flips it green (and fires a +// "recovered" alert) while the orphan still holds the backup lock and no backup can run. +// +// It is a sibling of the pid/status files in the identity dir, written with the same atomic +// rename idiom, and stays logging-free + stdlib-only like them. + +package health + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" +) + +// AbandonRecord is what the abandoning daemon leaves behind for its successor. PID is the +// orphan's pid -- the one an operator has to hunt for in D state -- and RID/TS identify the +// run it belonged to, so the degrade the next daemon reports can name its cause instead of +// being an unexplained failure. +type AbandonRecord struct { + // PID is the abandoned child's pid (0 when the process was never published). + PID int `json:"pid"` + // Start is the orphan's start time in clock ticks since boot (/proc//stat field 22), + // or 0 when it could not be read. It is the IDENTITY half of the pid, and without it the + // pid is not an identifier at all: the kernel recycles pid numbers WITHIN a boot, so a + // successor that only asked "is pid N alive" would keep the service-alive check DOWN + // forever the moment any unrelated long-lived process inherited the number -- a false RED + // that no run, no restart and no reboot could lift. The pair (pid, starttime) is unique + // for the life of a boot, and being a tick count rather than a wall-clock stamp it is + // immune to clock steps as well. + Start uint64 `json:"start,omitempty"` + // RID is the run id of the abandoned run, matching the /fail already sent on the + // backup-outcome check. + RID string `json:"rid,omitempty"` + // TS is the unix time in SECONDS of the abandon (the caller passes it; this package + // never reads the clock, like its siblings). + TS int64 `json:"ts"` +} + +// AbandonPath returns the abandoned-child marker path, a sibling of the status and pid files +// in the identity dir (same convention as StatusPath / DaemonPIDPath). +func AbandonPath(baseDir string) string { + return filepath.Join(baseDir, "identity", ".daemon_abandoned.json") +} + +// WriteAbandon persists the marker atomically: MkdirAll the identity dir, WriteFile a ".tmp" +// sibling at 0o600, then Rename over the final path so a reader sees either the old or the +// new file, never a partial one. +func WriteAbandon(baseDir string, rec AbandonRecord) error { + path := AbandonPath(baseDir) + dir := filepath.Dir(path) + if err := os.MkdirAll(dir, 0o750); err != nil { + return fmt.Errorf("create dir %s: %w", dir, err) + } + data, err := json.MarshalIndent(rec, "", " ") + if err != nil { + return fmt.Errorf("marshal abandon marker: %w", err) + } + tmp := path + ".tmp" + if err := os.WriteFile(tmp, append(data, '\n'), 0o600); err != nil { + return fmt.Errorf("write abandon marker: %w", err) + } + if err := os.Rename(tmp, path); err != nil { + _ = os.Remove(tmp) // best-effort cleanup so a failed rename leaves no stray ".tmp" + return fmt.Errorf("rename abandon marker: %w", err) + } + return nil +} + +// ReadAbandon returns the marker, or (nil, nil) when there is none -- the normal state. A +// present-but-unreadable marker is NOT an error either: its mere existence is the signal, so +// a corrupt file still yields a zero-valued record (pid 0, no rid) rather than being mistaken +// for "no abandon happened". Only a genuine read fault is returned. +func ReadAbandon(baseDir string) (*AbandonRecord, error) { + data, err := os.ReadFile(AbandonPath(baseDir)) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, fmt.Errorf("read abandon marker: %w", err) + } + var rec AbandonRecord + if len(data) > 0 { + _ = json.Unmarshal(data, &rec) // presence is the signal; unparseable contents degrade to the zero record + } + return &rec, nil +} + +// ClearAbandon removes the marker, lifting the degrade. A missing file is not an error, so +// this is idempotent. +func ClearAbandon(baseDir string) error { + if err := os.Remove(AbandonPath(baseDir)); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("remove abandon marker: %w", err) + } + return nil +} diff --git a/internal/health/abandon_test.go b/internal/health/abandon_test.go new file mode 100644 index 00000000..dd419f8f --- /dev/null +++ b/internal/health/abandon_test.go @@ -0,0 +1,64 @@ +package health + +import ( + "os" + "testing" +) + +// TestAbandonMarkerRoundTrip pins the three states the daemon distinguishes: no marker (the +// normal case), a marker it wrote itself, and a marker it has cleared after a run completed. +func TestAbandonMarkerRoundTrip(t *testing.T) { + base := t.TempDir() + + if rec, err := ReadAbandon(base); err != nil || rec != nil { + t.Fatalf("no marker must read as (nil, nil), got (%+v, %v)", rec, err) + } + if err := ClearAbandon(base); err != nil { + t.Fatalf("clearing an absent marker must be a no-op: %v", err) + } + + if err := WriteAbandon(base, AbandonRecord{PID: 4242, RID: "abc", TS: 1700000000}); err != nil { + t.Fatalf("WriteAbandon: %v", err) + } + rec, err := ReadAbandon(base) + if err != nil { + t.Fatalf("ReadAbandon: %v", err) + } + if rec == nil || rec.PID != 4242 || rec.RID != "abc" || rec.TS != 1700000000 { + t.Fatalf("marker round-trip lost data: %+v", rec) + } + if _, err := os.Stat(AbandonPath(base) + ".tmp"); !os.IsNotExist(err) { + t.Fatalf("the atomic write must leave no .tmp behind (err=%v)", err) + } + + if err := ClearAbandon(base); err != nil { + t.Fatalf("ClearAbandon: %v", err) + } + if rec, err := ReadAbandon(base); err != nil || rec != nil { + t.Fatalf("a cleared marker must read as (nil, nil), got (%+v, %v)", rec, err) + } +} + +// TestAbandonMarkerPresenceIsTheSignal guards the tolerance that matters: the daemon reads +// this file to decide whether to keep reporting the service-alive check DOWN. Unreadable +// CONTENTS must never be mistaken for "no abandon happened" -- that would silently re-green a +// host whose backups are dead, which is the whole failure this marker exists to prevent. +func TestAbandonMarkerPresenceIsTheSignal(t *testing.T) { + base := t.TempDir() + if err := WriteAbandon(base, AbandonRecord{PID: 1}); err != nil { + t.Fatalf("WriteAbandon: %v", err) + } + if err := os.WriteFile(AbandonPath(base), []byte("{not json"), 0o600); err != nil { + t.Fatalf("corrupt the marker: %v", err) + } + rec, err := ReadAbandon(base) + if err != nil { + t.Fatalf("a corrupt marker must not be an error: %v", err) + } + if rec == nil { + t.Fatal("a corrupt marker must still count as an abandon, not as its absence") + } + if rec.PID != 0 { + t.Fatalf("unparseable contents must degrade to the zero record, got %+v", rec) + } +} diff --git a/internal/health/reporter.go b/internal/health/reporter.go index ff3f1508..c23fb969 100644 --- a/internal/health/reporter.go +++ b/internal/health/reporter.go @@ -29,7 +29,7 @@ import ( // Ping suffixes on a healthchecks check URL (same identifier, different suffix). const ( suffixStart = "/start" // run started (pair with ?rid for duration) - suffixFail = "/fail" // definitive failure (used for the hang case) + suffixFail = "/fail" // definitive failure (a backup hang, or the alive degrade) suffixLog = "/log" // records a ping WITHOUT changing check state (test only) ) @@ -190,6 +190,26 @@ func (r *Reporter) RunHang(ctx context.Context, rid string, timeout time.Duratio return r.pingCheck(ctx, CheckKeyBackup, suffixFail, rid, body, "hang") } +// AliveDegraded pings /fail on the SERVICE-ALIVE check, driving it DOWN. It is the +// deliberate counterpart of Heartbeat: the daemon sends it when it has had to abandon a +// backup child the kernel will never let it reap -- once on its way out, and then in place +// of every heartbeat once systemd has restarted it, for as long as the abandoned child is +// still outstanding. Backups are dead throughout, and a host whose alive check stays green +// through that (or, worse, re-greens ten seconds later and fires a "recovered" alert) is a +// host nobody looks at. +// +// reason rides as the POST body so the monitor UI shows WHY the service degraded instead of +// an unexplained missed ping. It is a one-line diagnostic, NOT a log tail, so -- exactly +// like RunHang's timeout line -- it is not gated on SendLog. No rid: the alive check is not +// run-scoped and never opens a matching /start, so a ?rid on it would only corrupt the +// monitor's run-duration correlation. +func (r *Reporter) AliveDegraded(ctx context.Context, reason string) error { + if !r.HasAliveURL() { + return ErrNoAliveURL + } + return r.pingCheck(ctx, CheckKeyAlive, suffixFail, "", reason, "alive-degraded") +} + // ReportUpdate pings the "updates" check with the update-availability signal: // available==false -> /0 (up to date; the check stays UP/green), available==true -> /1 // (a non-zero "exit" so the check goes DOWN/red and fires the user's alerts), mirroring diff --git a/internal/health/reporter_test.go b/internal/health/reporter_test.go index 4424149b..ada096c5 100644 --- a/internal/health/reporter_test.go +++ b/internal/health/reporter_test.go @@ -242,3 +242,28 @@ func TestNewRunIDIsUUIDv4(t *testing.T) { seen[id] = true } } + +// TestAliveDegraded pins the alive-degrade contract: /fail on the ALIVE check (not the backup +// one), the reason in the body even with SendLog off, no rid (the alive check is not +// run-scoped and never opens a matching /start), and ErrNoAliveURL -- not a silent success -- +// when no alive URL was ever resolved. +func TestAliveDegraded(t *testing.T) { + cap, rep, done := newServer(t, 200, false) // SendLog off: the reason is a diagnostic, not a log tail + defer done() + if err := rep.AliveDegraded(context.Background(), "child unreapable"); err != nil { + t.Fatalf("AliveDegraded: %v", err) + } + g := cap.get() + if g.path != "/ping/alive/fail" { + t.Fatalf("alive-degraded path %q, want /ping/alive/fail", g.path) + } + if g.body != "child unreapable" { + t.Fatalf("alive-degraded body %q, want the reason even with SendLog off", g.body) + } + if g.query != "" { + t.Fatalf("alive-degraded should carry no rid, got query %q", g.query) + } + if err := NewReporter(Config{}).AliveDegraded(context.Background(), "x"); err != ErrNoAliveURL { + t.Fatalf("AliveDegraded with no url = %v, want ErrNoAliveURL", err) + } +} diff --git a/internal/health/status.go b/internal/health/status.go index 712cb2df..5f34c4f0 100644 --- a/internal/health/status.go +++ b/internal/health/status.go @@ -67,7 +67,14 @@ type PingRecord struct { // sensor panel is red for a failed backup instead of green (F09-02). It is orthogonal to // OK (which is only whether the ping transmitted): a perfectly transmitted /fail is // OK==true AND Down==true. Unused (false) for the alive heartbeat, whose liveness has no - // outcome. Omitted when false so old readers see byte-identical records on a downgrade. + // outcome -- and that stays true of the daemon's alive-degrade /fail on both of its paths. + // The exit-time degrade is transmitted but deliberately NOT recorded at all: a record + // would only refresh the liveness timestamp of a daemon that is at that moment dying. The + // periodic degrade a RESTARTED daemon sends while an abandoned child is still outstanding + // IS recorded, but as a plain liveness trace with Down false -- that daemon really is + // alive, no reader of this kind consults Down anyway (see sensors.go's alive row and + // Diagnose), and the DOWN signal rides the remote /fail where the alerting lives. Omitted + // when false so old readers see byte-identical records on a downgrade. Down bool `json:"down,omitempty"` } From 72bec0d0968a85b777f407336618952fff9fef7c Mon Sep 17 00:00:00 2001 From: tis24dev Date: Tue, 4 Aug 2026 04:12:17 +0200 Subject: [PATCH 28/50] docs: correct the D-state caveat and the alive-check description DAEMON.md claimed the daemon "reports the hang and moves on" for a child in uninterruptible sleep. That was the one case where it could not: the report sat downstream of a cmd.Run() that never returned. Replace it with what the code now does -- abandon, report both checks DOWN, exit for a systemd restart -- and with what an operator needs: where the marker lives, what lifts the degrade, why the restart gap is minutes rather than the nominal ten seconds, and that `ps -eo pid,stat,wchan,cmd | grep ' D'` is the fastest confirmation. HEALTHCHECKS.md's proxsave-alive row said the check only ever stops pinging. It is now also failed on purpose while an abandoned child is outstanding, which is the one situation where it reports DOWN for a daemon that is provably running. Closes the MED finding recorded against DAEMON.md:126. --- docs/DAEMON.md | 13 ++++++++++++- docs/HEALTHCHECKS.md | 2 +- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/docs/DAEMON.md b/docs/DAEMON.md index 41ee1631..a6ba9b6c 100644 --- a/docs/DAEMON.md +++ b/docs/DAEMON.md @@ -133,4 +133,15 @@ See [CONFIGURATION.md](CONFIGURATION.md) for the full variable reference and [CL ## Caveat: uninterruptible sleep (D state) -A backup child wedged in uninterruptible sleep on a dead mount cannot be killed even with `SIGKILL` (a kernel limit). The daemon still **reports the hang and moves on**, and the monitor's server-side `/start` plus grace catches it too; the `FS_IO_TIMEOUT` / `safefs` defenses are the layer below this watchdog. +A backup child wedged in uninterruptible sleep on a dead mount cannot be killed even with `SIGKILL`, and cannot be waited on either (both are kernel limits). The daemon gives the child its normal timeout, then `SIGTERM`, then `SIGKILL`, then 15 more seconds to actually be collected. If it still has not been, the child is **abandoned** and the daemon takes itself out of the way: + +1. Both checks go DOWN, in that order: `proxsave-backup` gets the hang report, then `proxsave-alive` is explicitly failed with the orphan's pid and run id in the body. The service-alive check must never stay green while backups are dead, so this is the one situation in which it reports DOWN for a daemon that is provably running. +2. A marker file, `/identity/.daemon_abandoned.json`, records the abandon so the fact outlives the process. +3. The daemon exits `4` (backup error) and **systemd restarts it** (`Restart=always`, `RestartSec=10`). The restart is what clears out the goroutines and file descriptors stranded behind a child that can never be reaped. Expect the gap to be minutes rather than the nominal ten seconds: the orphan is still in the unit's cgroup, so the stop job sits through its timeout waiting for a cgroup that cannot drain. +4. The restarted daemon reads the marker and keeps `proxsave-alive` DOWN instead of sending its usual heartbeat, so the outage does not look like it recovered ten seconds later. The orphan itself stays in D state; nothing in userspace can clear that. + +The degrade is lifted -- and the marker deleted -- as soon as anything shows backups are working again: a backup that completes while the orphan is gone (scheduled or your own `proxsave --backup`, which the daemon picks up through the usual handoff), the orphan disappearing on its own (re-checked on every heartbeat, so a mount that comes back recovers within one interval), or a reboot. It is deliberately not lifted by a failed run alone: the backup lock the orphan holds is checked *last*, after the directory and disk-space checks that the same dead mount fails first, so a run can fail without ever having got near the orphan. A run that finished with warnings (exit `1`) does count as completing -- it reached the end of the backup, and so passed the lock. + +If backups are administratively off (`BACKUP_ENABLED=false`) the marker is kept but the alive check is left alone -- with backups off nothing could ever lift the degrade, and `proxsave-backup` is already down on its own merits. + +Your fastest confirmation is `ps -eo pid,stat,wchan,cmd | grep ' D'`; the monitor's server-side `/start` plus grace catches the same run from the other side, and the `FS_IO_TIMEOUT` / `safefs` defenses are the layer below this watchdog. diff --git a/docs/HEALTHCHECKS.md b/docs/HEALTHCHECKS.md index 105b6937..fae077c1 100644 --- a/docs/HEALTHCHECKS.md +++ b/docs/HEALTHCHECKS.md @@ -27,7 +27,7 @@ The daemon reports four families of checks, each shown on the monitor as a | Check | When it pings | What it covers | |-------|---------------|----------------| -| `proxsave-alive` | immediately at daemon start, then every `HEALTHCHECK_HEARTBEAT_INTERVAL` | the daemon and the host are up. Stops when either dies, and the monitor alarms on the silence | +| `proxsave-alive` | immediately at daemon start, then every `HEALTHCHECK_HEARTBEAT_INTERVAL` | the daemon and the host are up. Stops when either dies, and the monitor alarms on the silence. It is also pinged `/fail` on purpose, with the reason in the body, while a backup child abandoned in uninterruptible sleep is outstanding -- see [DAEMON.md](DAEMON.md#caveat-uninterruptible-sleep-d-state) | | `proxsave-backup` | per run: `/start` at launch, then the run's exit code, or `/fail` on a hang | whether the backup ran and how it ended | | `proxsave-updates` | immediately at daemon start, then every `HEALTHCHECK_UPDATE_INTERVAL` | `/0` when up to date, `/1` when a newer release exists, so the check goes down and tells you to upgrade | | `proxsave-notify-` | after each daemon-supervised run, one per channel the backup attempted | whether that notification channel actually delivered | From 63311f59c9160c2eec135efe0dfe4757bd6276f7 Mon Sep 17 00:00:00 2001 From: tis24dev Date: Tue, 4 Aug 2026 04:37:49 +0200 Subject: [PATCH 29/50] fix(daemon): stop reading a pre-lock exit code as proof the lock was taken When an abandon marker is too corrupt to name a probe-able pid, the daemon falls back to the completed run's exit code as its only evidence that the orphan is no longer holding the backup lock. That set accepted ExitGenericError, and one pre-lock path reported it: an aborted encryption-recipient setup returned 1 from ensureBackupAgeRecipientsReady, which runBackupModeSteps reaches before configurePreBackupChecker and therefore long before RunPreBackupChecks runs the lock gate. ErrIdleTimeout wraps ErrInputAborted, so simply walking away from the encryption prompt was enough: the standalone handoff carried a 1 to the daemon, which deleted the marker and re-greened the service-alive check over an orphan that had not moved. That return is now ExitConfigError, matching the other failure of the same call one branch below, and a setup the user never completed really is a configuration failure. It is the only pre-lock producer of 1 in that file. The set was also under-inclusive in the other direction. The per-phase codes -- storage, verification, collection, archive, compression, encryption -- can only be minted by RunGoBackup's phases, which run after the pre-flight checks passed, so they DO prove the lock was taken. Refusing them stranded the degrade on a host whose backups fail for an unrelated reason: an unliftable false RED, the same failure the pid identity check exists to prevent. They now count. The keep branch for a marker with no usable pid also kept transmitting the startup note, "no backup has completed since", after runs had completed. It now restates what is still true, as the pid branch already did. --- cmd/proxsave/backup_mode.go | 13 ++++++++++- cmd/proxsave/daemon.go | 36 +++++++++++++++++++++++++---- cmd/proxsave/daemon_abandon_test.go | 33 ++++++++++++++++++++++++++ docs/DAEMON.md | 8 +++++-- 4 files changed, 82 insertions(+), 8 deletions(-) diff --git a/cmd/proxsave/backup_mode.go b/cmd/proxsave/backup_mode.go index 0e3f2df9..66c04d57 100644 --- a/cmd/proxsave/backup_mode.go +++ b/cmd/proxsave/backup_mode.go @@ -188,8 +188,19 @@ func ensureBackupAgeRecipientsReady(opts backupModeOptions, orch *orchestrator.O orchInitDone(err) if errors.Is(err, orchestrator.ErrAgeRecipientSetupAborted) { + // ExitConfigError, matching the other failure of this same call below, NOT + // ExitGenericError. This return is structurally pre-lock: runBackupModeSteps calls + // initializeBackupOrchestrator first and returns on its early error, so + // configurePreBackupChecker -- and with it RunPreBackupChecks, whose LAST gate is the + // backup lock -- is never reached. Reporting 1 made this indistinguishable from + // applyIssueExitCode's post-lock "clean run with warnings", and the daemon reads that + // code as proof a run reached the lock (exitProvesLockWasTaken): an operator who simply + // walked away from the encryption prompt -- ErrIdleTimeout wraps ErrInputAborted, which + // mapInputAbortToAgeAbort turns into this error -- would have cleared an abandoned-child + // marker and re-greened the service-alive check over a live orphan. A setup the user + // never completed is a configuration failure, which is what 2 means. logging.Warning("Encryption setup aborted by user. Exiting...") - return backupAgeRecipientEarlyError(err, types.ExitGenericError), types.ExitGenericError.Int() + return backupAgeRecipientEarlyError(err, types.ExitConfigError), types.ExitConfigError.Int() } logging.Error("ERROR: %v", err) diff --git a/cmd/proxsave/daemon.go b/cmd/proxsave/daemon.go index a6a0b01d..39ce6765 100644 --- a/cmd/proxsave/daemon.go +++ b/cmd/proxsave/daemon.go @@ -1315,6 +1315,12 @@ func (d *daemon) clearAbandonMarkerOnCompletedRun(reason string, exitCode int) { } if !exitProvesLockWasTaken(exitCode) { logging.Info("daemon: %s, but the marker names no pid this daemon can check and the run did not reach the backup lock, so nothing proves it was taken; keeping the service-alive check DOWN", reason) + // Same correction the pid branch above makes, for the same reason: the startup note + // says "no backup has completed since", a run just did, and every beat from here on + // would transmit that as fact. This branch can say less -- with no checkable pid there + // is nothing to point the operator at -- but it must not keep asserting something it + // now knows to be false. + d.setAbandonNote("a previous run abandoned a backup child and the marker names no pid this daemon can check; backups have run since, but none of them proved it reached the backup lock") return } d.clearAbandonMarker(reason) @@ -1349,12 +1355,32 @@ func (d *daemon) abandonNoteNow() string { // caution: on such a host it is a service-alive DOWN that no run, no restart and no reboot can // lift, which is the same unliftable false RED the pid identity check exists to prevent. // -// Everything else is a failure code and proves nothing -- a pre-flight gate failure returns -// ExitBackupError (cmd/proxsave/backup_execution.go, runPreBackupChecks), and a dead mount -// under the backup path is exactly what fails those gates before the lock is ever reached. -// ExitBackupSkipped never arrives here: both callers filter it first. +// The per-phase failure codes qualify for the opposite reason: they can ONLY be minted by +// RunGoBackup's phases (internal/orchestrator/backup_run_phases.go builds a BackupError with +// them, and cmd/proxsave/backup_execution.go returns backupErr.Code), and that runs strictly +// after RunPreBackupChecks succeeded -- which means the lock gate passed. A run that died in +// collection or encryption is a failed backup, but it is proof the orphan was not holding the +// lock. Refusing them would strand the degrade on a host whose backups fail for an unrelated +// reason: an unliftable false RED, the same failure the pid identity check exists to prevent. +// +// Everything else proves nothing. A pre-flight gate failure returns ExitBackupError, and a +// dead mount under the backup path is exactly what fails those gates before the lock is ever +// reached. ExitBackupSkipped never arrives here: both callers filter it first. +// +// The set is only sound because no PRE-lock path reports one of these codes. That is a real +// constraint on the rest of the tree, not an observation: the encryption-setup abort in +// cmd/proxsave/backup_mode.go returned ExitGenericError until it was found to be exactly such +// a path, and now returns ExitConfigError. Anything added here must be checked the same way. func exitProvesLockWasTaken(exitCode int) bool { - return exitCode == types.ExitSuccess.Int() || exitCode == types.ExitGenericError.Int() + switch exitCode { + case types.ExitSuccess.Int(), types.ExitGenericError.Int(), + types.ExitStorageError.Int(), types.ExitVerificationError.Int(), + types.ExitCollectionError.Int(), types.ExitArchiveError.Int(), + types.ExitCompressionError.Int(), types.ExitEncryptionError.Int(): + return true + default: + return false + } } // hostBootUnix reads the host's boot time from /proc/stat "btime", in Unix seconds. It diff --git a/cmd/proxsave/daemon_abandon_test.go b/cmd/proxsave/daemon_abandon_test.go index 41c2f5f1..d4be3f47 100644 --- a/cmd/proxsave/daemon_abandon_test.go +++ b/cmd/proxsave/daemon_abandon_test.go @@ -1707,3 +1707,36 @@ func TestDaemonFileCleanupCannotHoldTheExit(t *testing.T) { t.Fatal("the exit is hostage to the identity dir: on a wedged BaseDir the daemon never returns from run() and systemd never restarts it") } } + +// TestExitProvesLockWasTakenClassifiesEveryDocumentedCode pins the fallback evidence rule used +// when a marker is too corrupt to name a probe-able pid. The set is only sound while no +// PRE-lock path reports one of the accepted codes, which is a constraint on the whole tree +// rather than a local property: the encryption-setup abort in backup_mode.go reported +// ExitGenericError until it was found to be exactly such a path. Anything added to the accepted +// column has to be traced back to a producer that runs after RunPreBackupChecks. +func TestExitProvesLockWasTakenClassifiesEveryDocumentedCode(t *testing.T) { + // Accepted: reached the end of a backup, or failed in a phase that only RunGoBackup can + // reach -- both strictly after the lock gate passed. + for _, c := range []types.ExitCode{ + types.ExitSuccess, types.ExitGenericError, + types.ExitStorageError, types.ExitVerificationError, types.ExitCollectionError, + types.ExitArchiveError, types.ExitCompressionError, types.ExitEncryptionError, + } { + if !exitProvesLockWasTaken(c.Int()) { + t.Errorf("exit %d is only produced after the lock gate; it must count as proof", c.Int()) + } + } + // Rejected: every code a run can report WITHOUT having reached the lock. ExitConfigError is + // the one this list exists for -- it is what an aborted or timed-out encryption prompt now + // returns, and accepting it would clear a marker over a live orphan. + for _, c := range []types.ExitCode{ + types.ExitConfigError, types.ExitEnvironmentError, types.ExitBackupError, + types.ExitNetworkError, types.ExitPermissionError, types.ExitDiskSpaceError, + types.ExitPanicError, types.ExitSecurityError, types.ExitBackupSkipped, + types.ExitGuardsPending, + } { + if exitProvesLockWasTaken(c.Int()) { + t.Errorf("exit %d can be reported before the lock gate; it must not count as proof", c.Int()) + } + } +} diff --git a/docs/DAEMON.md b/docs/DAEMON.md index a6ba9b6c..0e8b84eb 100644 --- a/docs/DAEMON.md +++ b/docs/DAEMON.md @@ -101,7 +101,7 @@ WantedBy=multi-user.target ## On-disk state files -The daemon coordinates through five small files under `/identity/`, all written atomically (temp file then rename, mode `0600`) and deliberately not made immutable so they can be rewritten: +The daemon coordinates through six small files under `/identity/`, all written atomically (temp file then rename, mode `0600`) and deliberately not made immutable so they can be rewritten: | File | Purpose | |------|---------| @@ -110,8 +110,10 @@ The daemon coordinates through five small files under `/identity/`, al | `.healthcheck_status.json` | the last ping outcome per check, read back by the run phase to report real transmission; a corrupt file is quarantined to `.corrupt` and reset | | `.notify_results.json` | the backup child's per-channel notification severities, handed to the daemon to drive the `proxsave-notify-*` pings | | `.manual_backup_outcome.json` | a standalone run's outcome, handed off for the daemon to ping | +| `.daemon_abandoned.json` | a backup child the kernel would not let the daemon reap: the orphan's pid and start time, the run id, and when it happened. See [the D-state caveat](#caveat-uninterruptible-sleep-d-state) | `.daemon.pid` and `.daemon_info.json` are written at startup and removed on shutdown. +`.daemon_abandoned.json` is the exception that deliberately **survives** shutdown — that is its whole purpose — and is removed only once something shows backups can run again. ## Configuration keys (`backup.env`) @@ -140,7 +142,9 @@ A backup child wedged in uninterruptible sleep on a dead mount cannot be killed 3. The daemon exits `4` (backup error) and **systemd restarts it** (`Restart=always`, `RestartSec=10`). The restart is what clears out the goroutines and file descriptors stranded behind a child that can never be reaped. Expect the gap to be minutes rather than the nominal ten seconds: the orphan is still in the unit's cgroup, so the stop job sits through its timeout waiting for a cgroup that cannot drain. 4. The restarted daemon reads the marker and keeps `proxsave-alive` DOWN instead of sending its usual heartbeat, so the outage does not look like it recovered ten seconds later. The orphan itself stays in D state; nothing in userspace can clear that. -The degrade is lifted -- and the marker deleted -- as soon as anything shows backups are working again: a backup that completes while the orphan is gone (scheduled or your own `proxsave --backup`, which the daemon picks up through the usual handoff), the orphan disappearing on its own (re-checked on every heartbeat, so a mount that comes back recovers within one interval), or a reboot. It is deliberately not lifted by a failed run alone: the backup lock the orphan holds is checked *last*, after the directory and disk-space checks that the same dead mount fails first, so a run can fail without ever having got near the orphan. A run that finished with warnings (exit `1`) does count as completing -- it reached the end of the backup, and so passed the lock. +What lifts the degrade -- and deletes the marker -- is **the orphan being gone**, not a backup succeeding. The daemon re-checks it on every heartbeat and after every completed run, identifying the process by its pid *and* its start time, since the kernel recycles pid numbers. So a mount that comes back recovers within one heartbeat interval; a reboot clears it; and a run that completes while the orphan is still there does **not** clear it, whatever its exit code. Backups demonstrably working and `proxsave-alive` DOWN can therefore coexist, by design: the orphan is still holding a lock nobody can take from it. + +The exit code only matters in one fallback, when the marker is too corrupt to name a pid the daemon can check. There is nothing to probe, so the run's own outcome is the only evidence, and only a code that proves the run got *past* the lock counts: `0`, `1` (a clean run with warnings), or a per-phase failure such as a storage or encryption error, all of which are reached after the lock gate. A pre-flight failure -- the directory or disk-space check that the same dead mount fails first -- proves nothing and lifts nothing. If backups are administratively off (`BACKUP_ENABLED=false`) the marker is kept but the alive check is left alone -- with backups off nothing could ever lift the degrade, and `proxsave-backup` is already down on its own merits. From 7d637e9a2096bf83093ad6a14784bc506d23253d Mon Sep 17 00:00:00 2001 From: tis24dev Date: Tue, 4 Aug 2026 11:49:24 +0200 Subject: [PATCH 30/50] fix(cli): stop painting a pending-guards exit red in the final summary --cleanup-guards returns ExitGuardsPending (17) when the cleanup itself ran fine but guards are still in place, typically hidden under a live mount. Both exit-code tables call 17 "not a failure" and tell the operator to unmount and retry, and cleanup_guards_verdict.go logs that outcome at Warning. exitCodeSeverity did not know the code, so it fell through to severityError and the footer printed red: the same command telling the operator, in one breath, that nothing went wrong and that something did. Classified as a warning next to ExitBackupSkipped, which is there for the same reason. A genuine cleanup failure is ExitGenericError and stays red. --- cmd/proxsave/main_footer.go | 8 ++++++++ cmd/proxsave/main_footer_test.go | 5 +++++ 2 files changed, 13 insertions(+) diff --git a/cmd/proxsave/main_footer.go b/cmd/proxsave/main_footer.go index 29867def..8990ced0 100644 --- a/cmd/proxsave/main_footer.go +++ b/cmd/proxsave/main_footer.go @@ -219,6 +219,14 @@ func exitCodeSeverity(exitCode int, logger *logging.Logger) exitSeverity { // A benign skip (another backup running / disabled): non-blocking, colored yellow like a // warning, never green success nor red error (F09-03). return severityWarning + case exitCode == types.ExitGuardsPending.Int(): + // --cleanup-guards found guards it could not remove yet, typically hidden under a live + // mount. The cleanup itself succeeded, which is why cleanup_guards_verdict.go logs that + // outcome at Warning and both docs/CLI_REFERENCE.md and docs/TROUBLESHOOTING.md call 17 + // "not a failure". Without this case it fell through to severityError and the footer + // contradicted all three: red for an outcome the operator is told to act on, not to + // report as a bug. A genuine cleanup failure is ExitGenericError, which is separate. + return severityWarning default: return severityError } diff --git a/cmd/proxsave/main_footer_test.go b/cmd/proxsave/main_footer_test.go index 94f1f8d3..255855d8 100644 --- a/cmd/proxsave/main_footer_test.go +++ b/cmd/proxsave/main_footer_test.go @@ -44,6 +44,11 @@ func TestExitCodeSeverity(t *testing.T) { {"generic-error-with-warnings", types.ExitGenericError.Int(), warned, severityWarning}, {"config-error", types.ExitConfigError.Int(), clean, severityError}, {"config-error-with-warnings", types.ExitConfigError.Int(), warned, severityError}, + // 17 is a state, not a failure: cleanup_guards_verdict.go logs it at Warning and both + // exit-code tables say "not a failure". A red footer over it tells the operator to file + // a bug for an outcome whose remedy is to unmount and retry. + {"guards-pending", types.ExitGuardsPending.Int(), clean, severityWarning}, + {"guards-pending-with-warnings", types.ExitGuardsPending.Int(), warned, severityWarning}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { From bfc09f1370e544697b5b2ee72e8522081ec7de30 Mon Sep 17 00:00:00 2001 From: tis24dev Date: Tue, 4 Aug 2026 11:50:38 +0200 Subject: [PATCH 31/50] docs(cli): name all four benign exit codes, not two The exit-code note said 16 and 17 were the non-zero codes that do not mean something went wrong. docs/TROUBLESHOOTING.md names four -- 1, 16, 17 and 130 -- and so does the doc comment on TestExitCodeProseCountsTheBenignCodes. This page contradicted them and itself: it documents 130 as a cancellation two rows above, and the very next note says a cloud upload failure finishes the run with 1. Exit codes are a scripting contract, so an operator writing a wrapper from this page still paged on warning-only runs and on Ctrl+C. Aligned to the four-code list, keeping the --cleanup-guards caveat that 1 there means the cleanup itself failed rather than a benign warning. --- docs/CLI_REFERENCE.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/docs/CLI_REFERENCE.md b/docs/CLI_REFERENCE.md index dcf00d2f..69475c55 100644 --- a/docs/CLI_REFERENCE.md +++ b/docs/CLI_REFERENCE.md @@ -807,11 +807,15 @@ USE_COLOR=false proxsave | `17` | guards still in place | `--cleanup-guards` only. The cleanup itself ran fine, but the storage is still locked: guard mounts or immutable flags are left behind (typically hidden under a live mount), or the remaining count could not be confirmed. Also returned by `--cleanup-guards --dry-run` when it finds guards. Not a failure — unmount the datastore and retry | | `130` | interrupted | The run was cancelled with Ctrl+C (128 plus SIGINT) | -**Note**: `16` and `17` are the non-zero codes that do not mean something went wrong. A -wrapper of the form `proxsave --backup || alert` will page you every time two runs -overlap unless it excludes `16`. For `--cleanup-guards`, `17` is the one to act on but -not to report as a bug: `1` there means the cleanup itself failed, which is a different -remedy. +**Note**: `1`, `16`, `17` and `130` are the non-zero codes that do not mean something +went wrong: `1` is also what a run that succeeded with warnings returns, `16` means no +backup was performed for a benign reason, `17` means a guard cleanup ran fine but the +storage is still locked, and `130` means the run was cancelled by hand. Only `2` through +`15` are unambiguous failures. A wrapper of the form `proxsave --backup || alert` will +page you on warning-only runs, every time two runs overlap, and on anything you Ctrl+C, +unless it excludes them. For `--cleanup-guards`, `17` is the one to act on but not to +report as a bug — and `1` means the opposite of what it means elsewhere: there it is the +cleanup itself failing, which is a different remedy. **Note**: Cloud storage is non-critical. A cloud upload failure does **not** abort the run with a storage error (`5`): the local backup is kept, but the failure is recorded as a From 6f7516b4572d767da2583d9621f8672e11672d2e Mon Sep 17 00:00:00 2001 From: tis24dev Date: Tue, 4 Aug 2026 11:58:06 +0200 Subject: [PATCH 32/50] test(docs): make the benign-exit-code prose assertion able to fail TestExitCodeProseCountsTheBenignCodes searched the whole document for `1`, `16`, `17` and `130`. Every one of those already appears in the exit-code table that TestExitCodesAreDocumented requires, so the assertion was satisfied by the table alone: the sentence could be deleted outright and the test still passed. Verified by deleting it. The sentence is the part a reader acts on, and it is the half that went stale last time -- the table was corrected while the prose still counted the old number. Now the search is scoped to prose paragraphs, found structurally (blank-line blocks that are neither a table nor a heading) rather than by phrase, because the two documents word the claim differently and a phrase match would silently stop covering one of them. A paragraph must name every benign code, and where the sentence spells out a count, that count must agree. Both failure modes are covered: deleting the CLI_REFERENCE note now fails, and so does changing TROUBLESHOOTING's "Four of them" to a wrong number. --- cmd/proxsave/exit_codes_doc_drift_test.go | 65 +++++++++++++++++++++-- 1 file changed, 62 insertions(+), 3 deletions(-) diff --git a/cmd/proxsave/exit_codes_doc_drift_test.go b/cmd/proxsave/exit_codes_doc_drift_test.go index 1725da13..59f1fe94 100644 --- a/cmd/proxsave/exit_codes_doc_drift_test.go +++ b/cmd/proxsave/exit_codes_doc_drift_test.go @@ -121,10 +121,69 @@ func TestExitCodeProseCountsTheBenignCodes(t *testing.T) { t.Errorf("%s still carries the pre-17 claim %q", doc.name, stale) } } - for _, code := range benign { - if !strings.Contains(text, code) { - t.Errorf("%s never mentions benign exit code %s", doc.name, code) + + // Search the PROSE, not the whole file. Every benign code also appears in the + // exit-code table, so a whole-document search is satisfied by the table alone: the + // sentence could be deleted outright, or state the wrong set, and this test still + // passed. Verified by deleting it. + // + // The anchor is structural rather than textual because the two documents word the + // claim differently -- "do not mean something went wrong" here, "are not failures" + // there -- and a phrase match would silently stop covering one of them. + var carriers []string + for _, para := range exitCodeProsePara(text) { + if containsAll(para, benign) { + carriers = append(carriers, para) + } + } + if len(carriers) == 0 { + t.Errorf("%s has no prose paragraph naming every benign code %v; the table alone does not tell a reader which non-zero codes are safe to ignore", doc.name, benign) + continue + } + // If the sentence counts them, the count has to be right. This is the half that + // actually went stale last time: the table was corrected while the sentence still + // said the old number. + for _, para := range carriers { + if m := benignCountWord.FindStringSubmatch(para); m != nil && m[1] != "Four" { + t.Errorf("%s prose says %q of the non-zero codes are benign, but names %d of them", doc.name, m[1], len(benign)) + } + } + } +} + +// benignCountWord matches a spelled-out count in the benign-codes sentence, so a +// corrected table beside a stale "Three of them" is caught. +var benignCountWord = regexp.MustCompile(`\b(One|Two|Three|Four|Five|Six)\b of them`) + +// exitCodeProsePara splits a document into blank-line separated paragraphs and drops +// every block that is a table or a heading, leaving the prose a reader acts on. +func exitCodeProsePara(text string) []string { + var out []string + for _, para := range strings.Split(text, "\n\n") { + para = strings.TrimSpace(para) + if para == "" { + continue + } + structural := false + for _, line := range strings.Split(para, "\n") { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "|") || strings.HasPrefix(line, "#") { + structural = true + break } } + if !structural { + out = append(out, para) + } + } + return out +} + +func containsAll(s string, needles []string) bool { + for _, n := range needles { + if !strings.Contains(s, n) { + return false + } } + return true } From 0278ceb98c0a848179298336a11fb9f06268511b Mon Sep 17 00:00:00 2001 From: tis24dev Date: Tue, 4 Aug 2026 12:04:16 +0200 Subject: [PATCH 33/50] test(docs): fail on an exit-code constant the doc matcher cannot read TestExitCodesAreDocumented reads the exit codes out of the source rather than repeating them, so the list cannot drift. But the matcher only understands "Name Type = " and dropped anything else with a bare continue, and the only backstop was len(codes) < 10. With 18 codes declared, half of them could be rewritten as iota or as expressions and the check would still pass while quietly covering fewer codes -- the same drift this file exists to catch, reintroduced through its own blind spot. Unreadable specs are now collected and reported by name, so changing the declaration style forces whoever does it to teach the matcher instead of silently narrowing the contract. The weak count guard is replaced by len(codes) == 0. Verified: rewriting ExitGuardsPending as ExitBackupSkipped + 1 now fails with "ExitGuardsPending (value is not an integer literal)" instead of passing. --- cmd/proxsave/exit_codes_doc_drift_test.go | 31 ++++++++++++++++++++--- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/cmd/proxsave/exit_codes_doc_drift_test.go b/cmd/proxsave/exit_codes_doc_drift_test.go index 59f1fe94..2d2fb584 100644 --- a/cmd/proxsave/exit_codes_doc_drift_test.go +++ b/cmd/proxsave/exit_codes_doc_drift_test.go @@ -43,7 +43,14 @@ func TestExitCodesAreDocumented(t *testing.T) { t.Fatalf("parse exit_codes.go: %v", err) } + // Anything this matcher cannot read is recorded, never skipped quietly. The matcher only + // understands "Name Type = ", which is how exit_codes.go declares every code + // today. Rewrite one as an iota, an expression, or a grouped spec and it would vanish from + // codes -- and a code absent from codes is a code this test stops requiring in the docs, + // which is the exact drift the file exists to prevent, reintroduced through its own blind + // spot. Failing here forces whoever changes the declaration style to teach the matcher. codes := map[string]int{} + var unreadable []string for _, decl := range file.Decls { gen, ok := decl.(*ast.GenDecl) if !ok || gen.Tok != token.CONST { @@ -51,22 +58,38 @@ func TestExitCodesAreDocumented(t *testing.T) { } for _, spec := range gen.Specs { value, ok := spec.(*ast.ValueSpec) - if !ok || len(value.Names) != 1 || len(value.Values) != 1 { + if !ok { + unreadable = append(unreadable, fmt.Sprintf("%s: not a value spec", fset.Position(spec.Pos()))) + continue + } + names := make([]string, 0, len(value.Names)) + for _, n := range value.Names { + names = append(names, n.Name) + } + label := strings.Join(names, ",") + if len(value.Names) != 1 || len(value.Values) != 1 { + unreadable = append(unreadable, fmt.Sprintf("%s (%d name(s), %d value(s))", label, len(value.Names), len(value.Values))) continue } lit, ok := value.Values[0].(*ast.BasicLit) if !ok || lit.Kind != token.INT { + unreadable = append(unreadable, label+" (value is not an integer literal)") continue } n, convErr := strconv.Atoi(lit.Value) if convErr != nil { + unreadable = append(unreadable, fmt.Sprintf("%s (%q: %v)", label, lit.Value, convErr)) continue } - codes[value.Names[0].Name] = n + codes[names[0]] = n } } - if len(codes) < 10 { - t.Fatalf("found only %d exit-code constants; the matcher has gone stale", len(codes)) + if len(unreadable) > 0 { + t.Fatalf("%d exit-code constant(s) the matcher cannot read, so they are silently exempt from the documentation check: %s", + len(unreadable), strings.Join(unreadable, "; ")) + } + if len(codes) == 0 { + t.Fatalf("no exit-code constants found; the matcher has gone stale") } // The interrupted code lives in main (128 + SIGINT), not in the types package, but it // is part of the same published contract. From b7d00682c3c3cd5e79fd118dc89afaa427bb2a21 Mon Sep 17 00:00:00 2001 From: tis24dev Date: Tue, 4 Aug 2026 12:17:37 +0200 Subject: [PATCH 34/50] test(install): give the blank-Edit daemon case a blank base TestCollectInstallWizardDataCLIBlankEditKeepsStoredDefaults ran its second wizard against the same file as the first. runConfigWizardCLI ends in installer.WriteConfigFileAtomic, so by then that file held the minimal key set the first run wrote, including HEALTHCHECK_MODE=off and SCHEDULER_MODE=cron. The half named after a blank Edit was therefore editing a populated base, and its [off] assertion was reading back a stored value rather than the default wizardBlankBaseStandIn produces. Both halves now start from their own empty file, so each one tests what its name says. Deleting the stand-in still fails both, as before -- the coupling was real but indirect, through the config the first run wrote, not through the second run's own base. --- cmd/proxsave/install_test.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/cmd/proxsave/install_test.go b/cmd/proxsave/install_test.go index 518a845f..fbf14e2a 100644 --- a/cmd/proxsave/install_test.go +++ b/cmd/proxsave/install_test.go @@ -1025,10 +1025,15 @@ func TestCollectInstallWizardDataCLIBlankEditKeepsStoredDefaults(t *testing.T) { t.Fatalf("HealthcheckMode = %q, want off", result.HealthcheckMode) } - // Same base, but answering daemon: the healthcheck default must stay [off]. + // Same KIND of base -- blank -- but answering daemon: the healthcheck default must + // stay [off]. A FRESH file, not cfgFile: the run above ended in + // installer.WriteConfigFileAtomic, so cfgFile now holds the minimal key set including + // HEALTHCHECK_MODE=off. Reusing it would leave this half asserting the stored value + // instead of the stand-in, and deleting wizardBlankBaseStandIn would not fail it. + daemonCfgFile := createTempFile(t, "") daemonOutput := captureStdout(t, func() { reader := bufio.NewReader(strings.NewReader("2\nn\nn\nn\nn\nn\nn\ndaemon\n\n03:15\n")) - _, err = runConfigWizardCLI(context.Background(), reader, cfgFile, cfgFile+".tmp", "/opt/proxsave", nil) + _, err = runConfigWizardCLI(context.Background(), reader, daemonCfgFile, daemonCfgFile+".tmp", "/opt/proxsave", nil) }) if err != nil { t.Fatalf("runConfigWizardCLI (daemon) error: %v", err) From 9706a272b55148f8234301ed843c8175af787379 Mon Sep 17 00:00:00 2001 From: tis24dev Date: Tue, 4 Aug 2026 13:05:26 +0200 Subject: [PATCH 35/50] test(ui): give the Charm driver deadlines headroom for a busy machine The driver tests start a real bubbletea event loop and poll its render buffer every 10ms against a fixed wall-clock budget, so what those deadlines measure is how promptly the runtime schedules that loop -- not how long the logic takes. Alone, such a test finishes in a second or two against a 15s or 60s budget. Inside the full package it competes for the CPU, and the render can land after the budget expires with no bug anywhere. TestDashboardUpgradeScreen and TestDashboardDiagnosticNotConfiguredShowsNotice both failed that way: green alone, green in most full runs, red in the ones where the package took ~100s instead of ~38s because the machine was busy. Measured, not guessed: not leaked goroutines (2 alone, 4 after every dashboard test, 3 after the package), not a global left behind, not one polluting neighbour -- both halves of the family reproduce it -- and adding -v to the same command flipped the outcome. The mechanism to widen these was already here but keyed only off the race detector, which is the same problem under a different cause. The normal-run factor is now 4, doubled again below four processors. Widening is free on the success path: every poll returns as soon as its condition is met, so this costs nothing on a green run and only delays the report of a genuine hang, which go test still bounds itself. Verified against GOMAXPROCS=1, which reproduced the failure deterministically before this change. --- internal/uitest/deadline.go | 52 ++++++++++++++++++++++++------ internal/uitest/deadline_norace.go | 7 ++-- 2 files changed, 47 insertions(+), 12 deletions(-) diff --git a/internal/uitest/deadline.go b/internal/uitest/deadline.go index 2b012da5..b0e07120 100644 --- a/internal/uitest/deadline.go +++ b/internal/uitest/deadline.go @@ -2,16 +2,48 @@ // imported only from _test.go files and never linked into the production binary. package uitest -import "time" +import ( + "runtime" + "time" +) -// Deadline scales a base driver-test timeout by a race-aware factor. The Charm -// driver tests poll a render buffer until a screen/line appears; under the race -// detector the bubbletea event loop runs roughly an order of magnitude slower, so a -// fixed wall-clock deadline (e.g. 5s) can fire spuriously even though the logic is -// correct. Because those polls return as soon as the condition is met, a wider -// deadline is FREE on the success path - it only adds headroom before a genuine -// hang is finally reported. Use it ONLY for UI-render polling deadlines, never for -// tests that assert an operation's own timeout behavior. +// Deadline scales a base driver-test timeout by a slowness-aware factor. +// +// The Charm driver tests start a REAL bubbletea event loop and then poll its render +// buffer every 10ms until a screen or a line appears. That loop has to be SCHEDULED to +// render at all, so these deadlines measure how promptly the runtime gets round to it, +// not how long the logic takes. Run on its own, such a test finishes in a second or two +// against a 15s or 60s budget. Run inside the whole package it competes with everything +// else for the CPU, and the render can arrive after the budget has expired -- with no +// bug anywhere. +// +// That is not hypothetical. TestDashboardUpgradeScreen and +// TestDashboardDiagnosticNotConfiguredShowsNotice both timed out this way: green in +// isolation, green in most full runs, red in the ones where the package took ~100s +// instead of ~38s because the machine was busy with something else. It was measured +// down to the cause -- not leaked goroutines (2 alone, 4 after every dashboard test), +// not a global left behind, not one polluting neighbour (both halves of the family +// reproduce it), and adding -v to the same command was enough to flip the outcome. +// +// Widening is FREE on the success path: every poll returns the moment its condition is +// met, so a larger budget costs nothing on a green run and only delays the report of a +// genuine hang, which `go test` still bounds with its own panic timeout. So the factor +// is deliberately generous rather than tuned to the slowdown we happened to measure. +// +// The race detector gets its own multiplier for the same reason under a different +// cause: its instrumentation slows the event loop by roughly an order of magnitude. +// Use this ONLY for UI-render polling deadlines, never for tests that assert an +// operation's own timeout behavior. func Deadline(base time.Duration) time.Duration { - return base * time.Duration(raceScale) + return base * time.Duration(raceScale*cpuScale()) +} + +// cpuScale doubles the budget again when the event loop has few processors to be +// scheduled on -- a small CI runner, a pinned GOMAXPROCS, or a container quota. Below +// four, a driver test and the rest of the suite are effectively taking turns. +func cpuScale() int { + if runtime.GOMAXPROCS(0) < 4 { + return 2 + } + return 1 } diff --git a/internal/uitest/deadline_norace.go b/internal/uitest/deadline_norace.go index 9d31bef0..cbc114d1 100644 --- a/internal/uitest/deadline_norace.go +++ b/internal/uitest/deadline_norace.go @@ -2,5 +2,8 @@ package uitest -// raceScale leaves driver-test render-poll deadlines unchanged for normal runs. -const raceScale = 1 +// raceScale is the headroom a normal run gets. It is not 1: these deadlines bound how +// promptly the bubbletea event loop is scheduled, and a test that renders in a second +// on its own can miss a 15s budget while the rest of the package competes for the CPU. +// See Deadline -- the extra headroom costs nothing when the run is green. +const raceScale = 4 From 19239ee760a986898b39b3c3f129115dccf49329 Mon Sep 17 00:00:00 2001 From: tis24dev Date: Tue, 4 Aug 2026 13:17:32 +0200 Subject: [PATCH 36/50] test(ui): anchor the driver match offset where the push happened, not where it was read waitScreen set matchStart to buf.Len() at the moment the TEST goroutine pulled a screen title off the channel. The producer records that title from rootModel.Update on pushScreenMsg (internal/ui/shell/router.go), strictly before bubbletea calls View and writes the screen's bytes. So the two are ordered correctly at the source and then lost on the way out: whenever the reader was scheduled late -- exactly what a busy machine does -- the render had already landed, and matchStart started PAST the text the test was about to wait for. The poll could then never match. That is a lost update, not slowness, which is why TestDashboardUpgradeScreen and TestDashboardDiagnosticNotConfiguredShowsNotice consumed their entire deadline instead of failing fast, and why widening the deadline to 120s did not help. The offset now travels with the title, sampled in the event loop. With the ORIGINAL 15s/60s deadlines restored, GOMAXPROCS=1 -- which reproduced the failure on demand -- passes, and the package drops from 60-178s to 34s because no test burns a deadline any more. --- .../dashboard_support_consent_test.go | 4 +- cmd/proxsave/dashboard_support_test.go | 4 +- cmd/proxsave/dashboard_upgrade_test.go | 16 +++---- cmd/proxsave/newkey_charm_test.go | 45 ++++++++++++++----- 4 files changed, 46 insertions(+), 23 deletions(-) diff --git a/cmd/proxsave/dashboard_support_consent_test.go b/cmd/proxsave/dashboard_support_consent_test.go index 55fe25ee..465d0d9a 100644 --- a/cmd/proxsave/dashboard_support_consent_test.go +++ b/cmd/proxsave/dashboard_support_consent_test.go @@ -25,11 +25,11 @@ type supportFormRun struct { func startSupportForm(t *testing.T) *supportFormRun { t.Helper() - driver := &newkeyUIDriver{t: t, buf: &shell.SyncBuffer{}, pushes: make(chan string, 8)} + driver := &newkeyUIDriver{t: t, buf: &shell.SyncBuffer{}, pushes: make(chan screenPush, 8)} ctx, cancel := context.WithCancel(context.Background()) run := &supportFormRun{driver: driver, meta: make(chan support.Meta, 1), done: make(chan struct{})} driver.session = shell.StartObservedForTest(ctx, shell.Config{AppName: "ProxSave", Subtitle: "Dashboard"}, - driver.buf, func(title string) { driver.pushes <- title }) + driver.buf, func(title string) { driver.pushes <- screenPush{title: title, at: driver.buf.Len()} }) go func() { defer close(run.done) if meta, ok := runDashboardSupportForm(ctx, driver.session); ok { diff --git a/cmd/proxsave/dashboard_support_test.go b/cmd/proxsave/dashboard_support_test.go index e5213658..f6201d7e 100644 --- a/cmd/proxsave/dashboard_support_test.go +++ b/cmd/proxsave/dashboard_support_test.go @@ -42,11 +42,11 @@ func TestHandleSupportIntroSkipsWhenMetaProvided(t *testing.T) { // TestSupportFormIsOneScreen: the support form is a SINGLE screen ("Support") that shows the // consent note, BOTH input fields and the Start confirm together (not a sequence of screens). func TestSupportFormIsOneScreen(t *testing.T) { - driver := &newkeyUIDriver{t: t, buf: &shell.SyncBuffer{}, pushes: make(chan string, 8)} + driver := &newkeyUIDriver{t: t, buf: &shell.SyncBuffer{}, pushes: make(chan screenPush, 8)} ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) driver.session = shell.StartObservedForTest(ctx, shell.Config{AppName: "ProxSave", Subtitle: "Dashboard"}, - driver.buf, func(title string) { driver.pushes <- title }) + driver.buf, func(title string) { driver.pushes <- screenPush{title: title, at: driver.buf.Len()} }) done := make(chan struct{}) go func() { _, _ = runDashboardSupportForm(ctx, driver.session); close(done) }() diff --git a/cmd/proxsave/dashboard_upgrade_test.go b/cmd/proxsave/dashboard_upgrade_test.go index 79015944..00ab1f7f 100644 --- a/cmd/proxsave/dashboard_upgrade_test.go +++ b/cmd/proxsave/dashboard_upgrade_test.go @@ -115,11 +115,11 @@ func TestDashboardUpgradeScreen(t *testing.T) { } // Build an observed session eagerly (the shared seam creates it lazily via a flow). - driver := &newkeyUIDriver{t: t, buf: &shell.SyncBuffer{}, pushes: make(chan string, 64)} + driver := &newkeyUIDriver{t: t, buf: &shell.SyncBuffer{}, pushes: make(chan screenPush, 64)} ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) driver.session = shell.StartObservedForTest(ctx, shell.Config{AppName: "ProxSave", Subtitle: "Dashboard"}, - driver.buf, func(title string) { driver.pushes <- title }) + driver.buf, func(title string) { driver.pushes <- screenPush{title: title, at: driver.buf.Len()} }) done := make(chan struct{}) go func() { @@ -206,11 +206,11 @@ func TestDashboardUpgradeExternalCheckFailureIsWarning(t *testing.T) { dashboardUpgradeVersion = func() string { return "1.0.0" } dashboardUpgradeCheck = func(context.Context, *logging.Logger, string) *UpdateInfo { return nil } - driver := &newkeyUIDriver{t: t, buf: &shell.SyncBuffer{}, pushes: make(chan string, 64)} + driver := &newkeyUIDriver{t: t, buf: &shell.SyncBuffer{}, pushes: make(chan screenPush, 64)} ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) driver.session = shell.StartObservedForTest(ctx, shell.Config{AppName: "ProxSave", Subtitle: "Dashboard"}, - driver.buf, func(title string) { driver.pushes <- title }) + driver.buf, func(title string) { driver.pushes <- screenPush{title: title, at: driver.buf.Len()} }) done := make(chan struct{}) go func() { @@ -256,11 +256,11 @@ func TestDashboardUpgradeMenu(t *testing.T) { return &UpdateInfo{NewVersion: false, Latest: "1.0.0", Current: "1.0.0"} } - driver := &newkeyUIDriver{t: t, buf: &shell.SyncBuffer{}, pushes: make(chan string, 64)} + driver := &newkeyUIDriver{t: t, buf: &shell.SyncBuffer{}, pushes: make(chan screenPush, 64)} ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) driver.session = shell.StartObservedForTest(ctx, shell.Config{AppName: "ProxSave", Subtitle: "Dashboard"}, - driver.buf, func(title string) { driver.pushes <- title }) + driver.buf, func(title string) { driver.pushes <- screenPush{title: title, at: driver.buf.Len()} }) done := make(chan struct{}) go func() { @@ -330,11 +330,11 @@ func TestDashboardUpgradeRestartDaemonRelaunchNote(t *testing.T) { return health.DaemonState{ProcessAlive: true, Aligned: true, AlignChecked: true, StartTS: 1 << 60, Version: "9.9.9"} }) - driver := &newkeyUIDriver{t: t, buf: &shell.SyncBuffer{}, pushes: make(chan string, 64)} + driver := &newkeyUIDriver{t: t, buf: &shell.SyncBuffer{}, pushes: make(chan screenPush, 64)} ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) driver.session = shell.StartObservedForTest(ctx, shell.Config{AppName: "ProxSave", Subtitle: "Dashboard"}, - driver.buf, func(title string) { driver.pushes <- title }) + driver.buf, func(title string) { driver.pushes <- screenPush{title: title, at: driver.buf.Len()} }) done := make(chan struct{}) go func() { diff --git a/cmd/proxsave/newkey_charm_test.go b/cmd/proxsave/newkey_charm_test.go index af980ae4..98cff5b2 100644 --- a/cmd/proxsave/newkey_charm_test.go +++ b/cmd/proxsave/newkey_charm_test.go @@ -21,30 +21,50 @@ import ( // writes the recipient file; a dying UI program maps to the interactive // abort, not a hard failure. +// screenPush is one screen transition as the EVENT LOOP saw it: the title, and the +// buffer offset at that instant. +// +// The offset has to be sampled here, by the producer, and not by whoever reads this +// channel. shell.observeScreenPush is called from rootModel.Update on pushScreenMsg +// (internal/ui/shell/router.go), which is strictly BEFORE bubbletea calls View and +// writes the new screen's bytes -- so buf.Len() taken here is always just short of +// that render. A reader that sampled buf.Len() itself would sample it whenever it +// happened to be scheduled, which on a busy machine is after the render has already +// landed: matchStart would then start PAST the very text the test is waiting for, and +// the poll could never match. Not slowness -- a lost update. It is why +// TestDashboardUpgradeScreen burned its whole deadline instead of failing fast, and +// why GOMAXPROCS=1 reproduced it on demand while a loaded 12-core box only sometimes +// did. +type screenPush struct { + title string + at int +} + type newkeyUIDriver struct { t *testing.T buf *shell.SyncBuffer - pushes chan string + pushes chan screenPush session *shell.Session cancel context.CancelFunc // matchStart is the buffer offset of the current screen's render, updated by - // waitScreen on each screen transition. Output pollers (waitOutput / the - // per-test waitFor closures) match from here so they see the CURRENT screen's - // text and never a stale match from an earlier screen (e.g. a repeated - // "enter continue" from a prior run in a loop). It never rewinds (the buffer - // is append-only), so matching from it is always in range. + // waitScreen on each screen transition from the offset the PRODUCER recorded + // (see screenPush). Output pollers (waitOutput / the per-test waitFor closures) + // match from here so they see the CURRENT screen's text and never a stale match + // from an earlier screen (e.g. a repeated "enter continue" from a prior run in a + // loop). It never rewinds (the buffer is append-only), so matching from it is + // always in range. matchStart int } func installNewkeySessionSeam(t *testing.T) *newkeyUIDriver { t.Helper() - d := &newkeyUIDriver{t: t, buf: &shell.SyncBuffer{}, pushes: make(chan string, 64)} + d := &newkeyUIDriver{t: t, buf: &shell.SyncBuffer{}, pushes: make(chan screenPush, 64)} orig := newAgeSetupSession newAgeSetupSession = func(ctx context.Context, cfg shell.Config) *shell.Session { progCtx, cancel := context.WithCancel(ctx) d.cancel = cancel d.session = shell.StartObservedForTest(progCtx, cfg, d.buf, func(title string) { - d.pushes <- title + d.pushes <- screenPush{title: title, at: d.buf.Len()} }) return d.session } @@ -71,11 +91,14 @@ func (d *newkeyUIDriver) waitScreen(title string) { for { select { case got := <-d.pushes: - if got == title { + if got.title == title { // Anchor subsequent output pollers to this screen's render, so a // waitOutput/waitFor after a transition cannot match stale text left - // in the cumulative buffer by an earlier screen. - d.matchStart = d.buf.Len() + // in the cumulative buffer by an earlier screen. The offset comes from + // the push itself, sampled in the event loop before this screen + // rendered -- see screenPush. Re-reading buf.Len() here instead would + // skip past the render whenever this goroutine was scheduled late. + d.matchStart = got.at return } case <-deadline: From 5cb4147a5ba0f08b826da2e6220033bee7bebdc8 Mon Sep 17 00:00:00 2001 From: tis24dev Date: Tue, 4 Aug 2026 13:40:34 +0200 Subject: [PATCH 37/50] Revert "test(ui): give the Charm driver deadlines headroom for a busy machine" That change treated the symptom. The driver tests were not missing their budgets because the machine was slow: waitScreen sampled its match offset when the TEST goroutine read a screen-push rather than when the event loop emitted it, so a reader scheduled after the render began matching past the text it was waiting for and could never match at all. Fixed at the source in the previous commit. With the original 15s/60s budgets restored, GOMAXPROCS=1 -- which reproduced the failure on demand -- passes, and the full suite is green. Keeping the widening now only costs: a genuine hang would be reported after 120s instead of 15s, and the same suite ran 72s against 34s with the original budgets, because tests that legitimately exhaust a wait had theirs quadrupled too. Deadline keeps a note pointing at the real cause, so the next person to see a driver test time out does not widen these again chasing it. --- internal/uitest/deadline.go | 59 +++++++++--------------------- internal/uitest/deadline_norace.go | 7 +--- 2 files changed, 20 insertions(+), 46 deletions(-) diff --git a/internal/uitest/deadline.go b/internal/uitest/deadline.go index b0e07120..de2520ea 100644 --- a/internal/uitest/deadline.go +++ b/internal/uitest/deadline.go @@ -2,48 +2,25 @@ // imported only from _test.go files and never linked into the production binary. package uitest -import ( - "runtime" - "time" -) +import "time" -// Deadline scales a base driver-test timeout by a slowness-aware factor. +// Deadline scales a base driver-test timeout by a race-aware factor. The Charm +// driver tests poll a render buffer until a screen/line appears; under the race +// detector the bubbletea event loop runs roughly an order of magnitude slower, so a +// fixed wall-clock deadline (e.g. 5s) can fire spuriously even though the logic is +// correct. Because those polls return as soon as the condition is met, a wider +// deadline is FREE on the success path - it only adds headroom before a genuine +// hang is finally reported. Use it ONLY for UI-render polling deadlines, never for +// tests that assert an operation's own timeout behavior. // -// The Charm driver tests start a REAL bubbletea event loop and then poll its render -// buffer every 10ms until a screen or a line appears. That loop has to be SCHEDULED to -// render at all, so these deadlines measure how promptly the runtime gets round to it, -// not how long the logic takes. Run on its own, such a test finishes in a second or two -// against a 15s or 60s budget. Run inside the whole package it competes with everything -// else for the CPU, and the render can arrive after the budget has expired -- with no -// bug anywhere. -// -// That is not hypothetical. TestDashboardUpgradeScreen and -// TestDashboardDiagnosticNotConfiguredShowsNotice both timed out this way: green in -// isolation, green in most full runs, red in the ones where the package took ~100s -// instead of ~38s because the machine was busy with something else. It was measured -// down to the cause -- not leaked goroutines (2 alone, 4 after every dashboard test), -// not a global left behind, not one polluting neighbour (both halves of the family -// reproduce it), and adding -v to the same command was enough to flip the outcome. -// -// Widening is FREE on the success path: every poll returns the moment its condition is -// met, so a larger budget costs nothing on a green run and only delays the report of a -// genuine hang, which `go test` still bounds with its own panic timeout. So the factor -// is deliberately generous rather than tuned to the slowdown we happened to measure. -// -// The race detector gets its own multiplier for the same reason under a different -// cause: its instrumentation slows the event loop by roughly an order of magnitude. -// Use this ONLY for UI-render polling deadlines, never for tests that assert an -// operation's own timeout behavior. +// Do NOT widen these to chase a driver test that times out on a busy machine. That +// symptom was chased here once and the factor briefly raised; the cause was in the +// driver harness, not in the budget. waitScreen sampled its match offset when the +// TEST goroutine read a screen-push, not when the event loop emitted it, so a reader +// scheduled after the render started matching PAST the text it was waiting for and +// could never match at all. See screenPush in cmd/proxsave/newkey_charm_test.go. A +// poll that burns its ENTIRE deadline is that shape of bug; a genuinely slow machine +// fails late and irregularly instead. func Deadline(base time.Duration) time.Duration { - return base * time.Duration(raceScale*cpuScale()) -} - -// cpuScale doubles the budget again when the event loop has few processors to be -// scheduled on -- a small CI runner, a pinned GOMAXPROCS, or a container quota. Below -// four, a driver test and the rest of the suite are effectively taking turns. -func cpuScale() int { - if runtime.GOMAXPROCS(0) < 4 { - return 2 - } - return 1 + return base * time.Duration(raceScale) } diff --git a/internal/uitest/deadline_norace.go b/internal/uitest/deadline_norace.go index cbc114d1..9d31bef0 100644 --- a/internal/uitest/deadline_norace.go +++ b/internal/uitest/deadline_norace.go @@ -2,8 +2,5 @@ package uitest -// raceScale is the headroom a normal run gets. It is not 1: these deadlines bound how -// promptly the bubbletea event loop is scheduled, and a test that renders in a second -// on its own can miss a 15s budget while the rest of the package competes for the CPU. -// See Deadline -- the extra headroom costs nothing when the run is green. -const raceScale = 4 +// raceScale leaves driver-test render-poll deadlines unchanged for normal runs. +const raceScale = 1 From 9d692db1f5e70e2f6905c1e0fc5dbbea560241ec Mon Sep 17 00:00:00 2001 From: tis24dev Date: Tue, 4 Aug 2026 13:57:33 +0200 Subject: [PATCH 38/50] fix(install): treat a whitespace-only backup.env as blank, like an empty one ApplySchedulerTimeSeed guarded on an exact-empty base, so a backup.env holding nothing but a newline was seeded with SCHEDULER_TIME. That flips ApplyInstallData's editingExisting to true, defeats its blank->embedded-default substitution, and hands the operator a handful of mutated keys where they should have got the whole template -- while a 0-byte file one keystroke away got the template. A difference made of invisible characters, which nobody can predict from the outside. The guard is now strings.TrimSpace, so blank means blank. The comment that recorded this as a known residue is replaced by the reasoning. The cost is deliberate: with such a file, Edit no longer offers the host's crontab run time as the "Run at" default. Nothing then claims otherwise -- adoptCronRunTimeIntoBase logs its adoption note only when the seed actually changed the base, comparing the two values, so a discarded seed stays silent. A comments-only file is NOT blank and must not be: that is content the operator wrote, and Edit keeps treating it as the existing configuration. Both halves are pinned by TestApplySchedulerTimeSeedWhitespaceBase. --- internal/installer/existing_config.go | 31 +++++++++++++++------- internal/installer/existing_config_test.go | 27 ++++++++++++++++--- 2 files changed, 46 insertions(+), 12 deletions(-) diff --git a/internal/installer/existing_config.go b/internal/installer/existing_config.go index f0e193ca..09e8027c 100644 --- a/internal/installer/existing_config.go +++ b/internal/installer/existing_config.go @@ -132,16 +132,29 @@ func BaseTemplateOrDefault(base string) string { // offers the host's real time instead of the 02:00 template default. It writes // nothing to disk. // -// The base=="" guard is the CLI's existing guard and is deliberately an -// EXACT-empty test, not strings.TrimSpace. Without it, seeding a "" base -// produces "\nSCHEDULER_TIME=HH:MM", which flips ApplyInstallData's -// editingExisting to true, defeats its blank->embedded-default substitution and -// writes a gutted config. KNOWN RESIDUE: a whitespace-only base is != "", so it -// is still mirrored into and still gutted; fixing that requires a TrimSpace test -// that would also change the CLI (a whitespace-only backup.env + Edit would stop -// adopting the crontab time), so it is out of scope here. +// The blank-base guard is load-bearing. Seeding a blank base produces +// "\nSCHEDULER_TIME=HH:MM", which flips ApplyInstallData's editingExisting to +// true, defeats its blank->embedded-default substitution, and writes a gutted +// config: the operator gets a handful of mutated keys where they should have got +// the whole template. +// +// The test is strings.TrimSpace, not an exact-empty comparison, so a +// whitespace-only backup.env is blank for this purpose exactly like an empty one. +// It used to be exact-empty, and the two then diverged on a single invisible +// character: an empty file produced the full template on Edit, a file holding one +// newline produced the gutted one. Nobody can predict that from the outside, and +// there is nothing in a whitespace-only file worth preserving. +// +// The cost is deliberate and small: with such a file, Edit no longer offers the +// host's crontab run time as the "Run at" default, falling back to the template's. +// Nothing then claims otherwise -- adoptCronRunTimeIntoBase logs its adoption note +// only when the seed actually changed the base (it compares the two values), so a +// discarded seed stays silent instead of promising a time it did not apply. +// +// A file that holds comments is NOT blank here, and must not be: that is content +// the operator wrote, and Edit keeps treating it as the existing configuration. func ApplySchedulerTimeSeed(base, hhmm string) string { - if hhmm == "" || base == "" { + if hhmm == "" || strings.TrimSpace(base) == "" { return base } return setEnvValue(base, "SCHEDULER_TIME", hhmm) diff --git a/internal/installer/existing_config_test.go b/internal/installer/existing_config_test.go index f0a304fb..a2c988a6 100644 --- a/internal/installer/existing_config_test.go +++ b/internal/installer/existing_config_test.go @@ -123,9 +123,9 @@ func TestExistingConfigPresent(t *testing.T) { } } -// TestApplySchedulerTimeSeedEmptyBase pins S3: the mirror keeps the CLI's -// exact-empty guard. Without it a "" base becomes "\nSCHEDULER_TIME=HH:MM", -// which flips ApplyInstallData's editingExisting to true, defeats its +// TestApplySchedulerTimeSeedEmptyBase pins S3: the mirror keeps its blank-base +// guard. Without it a blank base becomes "\nSCHEDULER_TIME=HH:MM", which flips +// ApplyInstallData's editingExisting to true, defeats its // blank->embedded-default substitution and writes a gutted config. func TestApplySchedulerTimeSeedEmptyBase(t *testing.T) { if got := ApplySchedulerTimeSeed("", "21:00"); got != "" { @@ -136,6 +136,27 @@ func TestApplySchedulerTimeSeedEmptyBase(t *testing.T) { } } +// TestApplySchedulerTimeSeedWhitespaceBase pins the half that used to escape. +// The guard was an exact-empty comparison, so a backup.env holding nothing but a +// newline was seeded, turned editingExisting on, and got the gutted config -- while +// a 0-byte file one keystroke away got the full template. There is nothing in a +// whitespace-only file to preserve, and no operator can predict a difference made +// of invisible characters. +// +// A comments-only base is the deliberate other side of that line: it is content the +// operator wrote, so it stays an existing configuration and is still seeded. +func TestApplySchedulerTimeSeedWhitespaceBase(t *testing.T) { + for _, base := range []string{" ", "\n", "\n\n", " \t\n "} { + if got := ApplySchedulerTimeSeed(base, "21:00"); got != base { + t.Errorf("whitespace-only base %q must be left alone like an empty one, got %q", base, got) + } + } + const commented = "# SCHEDULER_MODE=cron\n" + if got := ApplySchedulerTimeSeed(commented, "21:00"); got == commented { + t.Errorf("a comments-only base is real content and must still be seeded, got %q", got) + } +} + func TestApplySchedulerTimeSeedMirrorsTime(t *testing.T) { got := ApplySchedulerTimeSeed("SCHEDULER_MODE=cron\n", "21:00") if !strings.Contains(got, "SCHEDULER_TIME=21:00") { From e9ddb93e600581cc5f0e0f8ede7e3c587348f316 Mon Sep 17 00:00:00 2001 From: tis24dev Date: Tue, 4 Aug 2026 14:18:01 +0200 Subject: [PATCH 39/50] fix(restore): write the pre-restore safety archive owner-only createSafetyBackup opened the archive with safetyFS.Create, i.e. 0666&^umask, which is 0644 on a stock host. That archive is the copy of everything the restore is about to overwrite, so on a PVE or PBS node it carries /etc/shadow, /etc/pve/priv material and access-control config in the clear. It is written into /tmp/proxsave, which is 0755 and shared with every local account, and it is deliberately never deleted because it IS the rollback. World-readable was not a brief window, it was the steady state -- and the same is true of the four network/firewall/HA/access-control rollback tarballs, which take the same path. Opened 0600 instead. docs/ENCRYPTION.md documented the old mode and the exposure it caused, so it is corrected in the same commit rather than left to contradict the code. The archive's name and size are still visible: /tmp/proxsave stays 0755, which other flows depend on, and tightening it is a separate decision. --- docs/ENCRYPTION.md | 7 ++-- internal/orchestrator/backup_safety.go | 8 ++++- internal/orchestrator/backup_safety_test.go | 39 +++++++++++++++++++++ 3 files changed, 50 insertions(+), 4 deletions(-) diff --git a/docs/ENCRYPTION.md b/docs/ENCRYPTION.md index 07b9b7f9..0fdc5c11 100644 --- a/docs/ENCRYPTION.md +++ b/docs/ENCRYPTION.md @@ -103,9 +103,10 @@ The same applies in reverse, and the restore side is worse. `proxsave --decrypt` Nothing sweeps it either, since it is not registered. A restore also leaves its rollback and safety tarballs (`restore_backup_`, `network_rollback_backup_`, `firewall_rollback_backup_`, `ha_rollback_backup_`, `pve_access_control_rollback_backup_`, each `_.tar.gz`) -deliberately in place, and those are written **mode 0644 directly in the root**, not inside a -`0700` directory, so on a stock host any local user can read them. Clean them up yourself -once a restore has settled. +deliberately in place — they are the rollback. Those are written **mode 0600**, so their +contents are not readable by other local users even though `/tmp/proxsave` itself is `0755` +and shared; their names and sizes still are. Clean them up yourself once a restore has +settled. If you decrypt by hand with the `age` CLI, your own output is plaintext too: pipe it rather than land it on a shared filesystem. diff --git a/internal/orchestrator/backup_safety.go b/internal/orchestrator/backup_safety.go index cd24936a..d95ca035 100644 --- a/internal/orchestrator/backup_safety.go +++ b/internal/orchestrator/backup_safety.go @@ -56,7 +56,13 @@ func createSafetyBackup(logger *logging.Logger, selectedCategories []Category, d logger.Info("Creating %s of current configuration...", strings.ToLower(desc)) logger.Debug("%s will be saved to: %s", desc, backupArchive) - file, err := safetyFS.Create(backupArchive) + // 0600, not safetyFS.Create's 0666&^umask (0644 on a stock host). This archive is + // the pre-restore copy of whatever is about to be overwritten, so on a PVE or PBS + // node it holds /etc/shadow, /etc/pve/priv material and access-control config in + // the clear. It is written into /tmp/proxsave, which is 0755 and shared, and it is + // deliberately NOT deleted afterwards -- it is the rollback. World-readable was + // therefore not a brief window but the steady state. + file, err := safetyFS.OpenFile(backupArchive, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600) if err != nil { return nil, fmt.Errorf("create backup archive: %w", err) } diff --git a/internal/orchestrator/backup_safety_test.go b/internal/orchestrator/backup_safety_test.go index 9816028d..0347e89e 100644 --- a/internal/orchestrator/backup_safety_test.go +++ b/internal/orchestrator/backup_safety_test.go @@ -2419,3 +2419,42 @@ func TestBackupDirectory_WithMixedContent(t *testing.T) { } } } + +// TestCreateSafetyBackupArchiveIsOwnerOnly pins the permission on the pre-restore +// archive. It is the copy of everything the restore is about to overwrite, so on a PVE +// or PBS node it carries /etc/shadow, /etc/pve/priv material and access-control config +// in the clear. It lands in /tmp/proxsave, which is 0755 and shared with every local +// user, and it is deliberately never deleted -- it IS the rollback. safetyFS.Create +// would open it 0666&^umask, i.e. 0644 on a stock host, so any local account could +// read the node's secrets at leisure. +func TestCreateSafetyBackupArchiveIsOwnerOnly(t *testing.T) { + fake := NewFakeFS() + t.Cleanup(func() { _ = os.RemoveAll(fake.Root) }) + origFS := safetyFS + safetyFS = fake + t.Cleanup(func() { safetyFS = origFS }) + + fixed := time.Date(2024, time.March, 1, 15, 4, 5, 0, time.UTC) + origNow := safetyNow + safetyNow = func() time.Time { return fixed } + t.Cleanup(func() { safetyNow = origNow }) + + destRoot := "/restore-target" + if err := fake.AddFile(filepath.Join(destRoot, "etc/shadow"), []byte("root:$6$secret")); err != nil { + t.Fatalf("add shadow: %v", err) + } + + result, err := CreateSafetyBackup(logging.New(types.LogLevelError, false), + []Category{{ID: "accounts", Paths: []string{"./etc/shadow"}}}, destRoot) + if err != nil { + t.Fatalf("CreateSafetyBackup error: %v", err) + } + + info, err := fake.Stat(result.BackupPath) + if err != nil { + t.Fatalf("stat archive: %v", err) + } + if perm := info.Mode().Perm(); perm != 0o600 { + t.Fatalf("safety archive is %04o, want 0600: it holds /etc/shadow under a world-readable /tmp/proxsave and is never deleted", perm) + } +} From cbf86137b816322d94d438a8c7bf466435625736 Mon Sep 17 00:00:00 2001 From: tis24dev Date: Tue, 4 Aug 2026 14:27:35 +0200 Subject: [PATCH 40/50] docs(encryption): give the manual decryption recipes a private workspace Every emergency and verification recipe opened its scratch directory with mkdir -p, which leaves it 0755 under a shared /tmp, and then landed the decrypted archive either in it or in the operator's cwd. That plaintext is the node's whole configuration -- /etc/shadow, /etc/pve/priv and the rest -- so following the documented procedure exposed it to every local account for as long as it sat there, and two of the recipes never said to delete it. All six sites now use install -d -m 700, which also corrects a directory an earlier run of these recipes already created 0755. The plaintext is kept inside that directory rather than in the cwd, and each recipe ends by removing it. --- docs/ENCRYPTION.md | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/docs/ENCRYPTION.md b/docs/ENCRYPTION.md index 0fdc5c11..5a088ebf 100644 --- a/docs/ENCRYPTION.md +++ b/docs/ENCRYPTION.md @@ -225,9 +225,13 @@ ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIExampleSSHpublicKeyForAgeRecipient > > ```bash > # With bundling left at its default the raw .age is not on disk: untar the bundle first. -> mkdir -p /tmp/emergency +> # install -d -m 700, not mkdir -p: the decrypted archive lands in this directory, and +> # /tmp is shared, so a 0755 workspace hands the node's configuration to every local +> # account. Keep the plaintext inside it and delete it when you are done. +> install -d -m 700 /tmp/emergency > tar -xf -backup-YYYYMMDD-HHMMSS.tar.xz.age.bundle.tar -C /tmp/emergency -> age -d -i ~/.ssh/id_ed25519 -o backup.tar.xz /tmp/emergency/-backup-YYYYMMDD-HHMMSS.tar.xz.age +> age -d -i ~/.ssh/id_ed25519 -o /tmp/emergency/backup.tar.xz /tmp/emergency/-backup-YYYYMMDD-HHMMSS.tar.xz.age +> rm -rf /tmp/emergency # once you have what you needed > ``` ### Interactive Wizard @@ -369,10 +373,13 @@ unwrap the bundle first (see [Emergency Decryption Without Configuration](#emergency-decryption-without-configuration)): ```bash -mkdir -p /tmp/emergency +# 0700 workspace: the decrypted archive below is plaintext and /tmp is shared. +install -d -m 700 /tmp/emergency tar -xf -backup-YYYYMMDD-HHMMSS.tar.xz.age.bundle.tar -C /tmp/emergency age --decrypt -i /path/to/age-keys.txt \ - /tmp/emergency/-backup-YYYYMMDD-HHMMSS.tar.xz.age > -backup-YYYYMMDD-HHMMSS.tar.xz + /tmp/emergency/-backup-YYYYMMDD-HHMMSS.tar.xz.age \ + > /tmp/emergency/-backup-YYYYMMDD-HHMMSS.tar.xz +rm -rf /tmp/emergency # once you have what you needed ``` > **Passphrase recipients are not native age passphrases.** A passphrase recipient @@ -529,8 +536,9 @@ tar -tf -backup-YYYYMMDD-HHMMSS.tar.xz.age.bundle.tar # -backup-YYYYMMDD-HHMMSS.tar.xz.age # 2. Unwrap. The bundle is an uncompressed tar with basename-only entries, -# so extract it into a directory of your own. -mkdir -p /tmp/emergency +# so extract it into a directory of your own. 0700, not the 0755 mkdir -p would +# give it: everything from step 4 on is plaintext and /tmp is shared. +install -d -m 700 /tmp/emergency tar -xf -backup-YYYYMMDD-HHMMSS.tar.xz.age.bundle.tar -C /tmp/emergency cd /tmp/emergency @@ -542,9 +550,13 @@ sha256sum -c -backup-YYYYMMDD-HHMMSS.tar.xz.age.sha256 age --decrypt -i /path/to/age-keys.txt \ -backup-YYYYMMDD-HHMMSS.tar.xz.age > -backup-YYYYMMDD-HHMMSS.tar.xz -# 5. Extract the inner archive -mkdir -p /tmp/emergency-restore +# 5. Extract the inner archive. This is the node's configuration in the clear -- +# /etc/shadow, /etc/pve/priv and the rest -- so the destination is 0700 too. +install -d -m 700 /tmp/emergency-restore tar -xf -backup-YYYYMMDD-HHMMSS.tar.xz -C /tmp/emergency-restore + +# 6. When you are done, remove both workspaces: nothing else will. +rm -rf /tmp/emergency /tmp/emergency-restore ``` Notes on this recipe: @@ -602,7 +614,9 @@ compression only when it is given a **file name**, so piping a compressed stream used. ```bash -mkdir -p /tmp/emergency +# 0700: archive.inner below is the whole configuration in the clear, and although +# this recipe deletes it, /tmp is shared for as long as the check runs. +install -d -m 700 /tmp/emergency tar -xOf -backup-YYYYMMDD-HHMMSS.tar.xz.age.bundle.tar \ -backup-YYYYMMDD-HHMMSS.tar.xz.age \ | age --decrypt -i /path/to/age-keys.txt > /tmp/emergency/archive.inner From 2eacabd98bac648c8f4de4d08dbaaabd01743319 Mon Sep 17 00:00:00 2001 From: tis24dev Date: Tue, 4 Aug 2026 14:31:36 +0200 Subject: [PATCH 41/50] docs(encryption): stop implying the recipient-file salt comment can rescue derivation The key-derivation section said the "# passphrase-salt:" comment "is only consulted once the sibling is gone", which reads as a fallback that keeps a lost passphrase.salt recoverable. The migration table two hundred lines earlier says the opposite, and the migration table is right. readPassphraseSalt, the only reader of that comment, has exactly one caller: passphraseSaltForManifest. So the comment feeds the archive manifest and nothing else. Derivation goes through getOrCreatePassphraseSalt, which reads the sibling alone and, when it is absent, generates a fresh random salt and derives a DIFFERENT recipient without warning. Reworded to say which of the two the comment can still do and which it cannot. Also spelled out why the sibling wins: the backup rewrites the comment from it before reading it back, so precedence comes from that copy, not from a comparison. --- docs/ENCRYPTION.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ENCRYPTION.md b/docs/ENCRYPTION.md index 5a088ebf..726233b2 100644 --- a/docs/ENCRYPTION.md +++ b/docs/ENCRYPTION.md @@ -633,7 +633,7 @@ rm -f /tmp/emergency/archive.inner ### Encryption Implementation - **Algorithm**: ChaCha20-Poly1305 (AEAD) with X25519 ECDH -- **Key derivation**: scrypt (N=2^15, r=8, p=1) for passphrases. The current scheme uses a **per-installation random salt** (v2), generated once, stored `0600` at `identity/age/passphrase.salt`, mirrored as the `# passphrase-salt:` line inside the recipient file (every backup rewrites that comment from the sibling, so the sibling wins whenever the two differ; the comment is only consulted once the sibling is gone), and embedded in each manifest as `passphrase_salt` so the passphrase alone can re-derive the recipient on any host. At decrypt ProxSave tries salts in order: the manifest's per-install salt first, then two fixed legacy namespaces (`proxsave/age-passphrase/v1`, then the pre-rebrand `proxmox-backup-go/age-passphrase/v1`), so archives from older versions and from before the rename stay decryptable. +- **Key derivation**: scrypt (N=2^15, r=8, p=1) for passphrases. The current scheme uses a **per-installation random salt** (v2), generated once, stored `0600` at `identity/age/passphrase.salt`, mirrored as the `# passphrase-salt:` line inside the recipient file (every backup rewrites that comment from the sibling before reading it back, so the sibling wins whenever the two differ; once the sibling is gone the comment still supplies the salt stamped into new manifests, but it can **not** re-derive the recipient — the setup wizard reads the sibling alone and mints a fresh random salt when it is missing, yielding a different recipient), and embedded in each manifest as `passphrase_salt` so the passphrase alone can re-derive the recipient on any host. At decrypt ProxSave tries salts in order: the manifest's per-install salt first, then two fixed legacy namespaces (`proxsave/age-passphrase/v1`, then the pre-rebrand `proxmox-backup-go/age-passphrase/v1`), so archives from older versions and from before the rename stay decryptable. - **Random nonces**: Unique per encryption operation - **Authentication**: Poly1305 MAC prevents tampering From cfc998e42b6d9f362f208f06e02c40da09b21e10 Mon Sep 17 00:00:00 2001 From: tis24dev Date: Tue, 4 Aug 2026 14:37:56 +0200 Subject: [PATCH 42/50] docs(encryption): make the leftover-cleanup recipe safe to run, and split two warnings The cleanup block was four unscoped rm -rf globs under a bare "check by hand". Run on a working host they take the staging directory out from under a backup, a decrypt or a restore in progress, and the *_backup_*.tar.gz glob is not leftovers at all -- those tarballs are the restore's rollback, which is exactly what an operator reaches for when a restore has gone wrong. The recipe now gates on pgrep, keeps the ls, and deletes by name: the patterns stay as identification of what each entry is, with one worked example, instead of being presented as commands to paste. The rollback tarballs carry their own condition. Separately, MD028: a bare blank line sat between the passphrase-salt warning and the SSH-keys one. They are different warnings, so they are separated by a lead-in sentence rather than merged into one blockquote. --- docs/ENCRYPTION.md | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/docs/ENCRYPTION.md b/docs/ENCRYPTION.md index 726233b2..b5601c3c 100644 --- a/docs/ENCRYPTION.md +++ b/docs/ENCRYPTION.md @@ -83,12 +83,26 @@ host `/var/run` is tmpfs, so a crash followed by a reboot loses the record and t is never swept. Check by hand after an unclean shutdown, and note that the sweep only ever covers backup staging: +Every path below belongs to a **live** run until that run ends, so check first and delete by +name. A glob run against a working host takes the staging directory out from under a backup, +a decrypt or a restore in progress — and the safety tarballs are not leftovers at all, they +are the restore's rollback. + ```bash +# 1. Is anything running? If this prints a PID, stop here. +pgrep -a -x proxsave + +# 2. Look at what is actually there, with dates. ls -la /tmp/proxsave/ -rm -rf /tmp/proxsave/proxsave-* # backup staging, from a killed backup -rm -rf /tmp/proxsave/proxmox-decrypt-* # decrypt staging: a FULLY DECRYPTED archive -rm -rf /tmp/proxsave/restore-stage-* # restore staging: plaintext shadow and pve priv -rm -f /tmp/proxsave/*_backup_*.tar.gz # restore safety tarballs, once the restore is settled + +# 3. Delete the specific entries you judged stale, substituting the real names from +# step 2. The patterns identify what each one is; they are not meant to be run as +# globs on a host that is still working. +# proxsave-* backup staging, from a killed backup +# proxmox-decrypt-* decrypt staging: a FULLY DECRYPTED archive +# restore-stage-* restore staging: plaintext shadow and pve priv +# *_backup_*.tar.gz the restore's rollback -- only once that restore has settled +rm -rf /tmp/proxsave/restore-stage-20260803-120000_1 ``` Two practical consequences. `/tmp` needs room for a full uncompressed copy of everything @@ -217,6 +231,8 @@ ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIExampleSSHpublicKeyForAgeRecipient > **new random salt** and derives a **different** recipient, without a warning: the comment > is not consulted there. +SSH recipients carry their own asymmetry, in the opposite direction: + > **SSH keys encrypt, but ProxSave cannot decrypt with them.** `proxsave --decrypt` and `proxsave --restore` accept only an `AGE-SECRET-KEY-...` identity or a passphrase. Paste an SSH private key at the prompt and it is hashed as a passphrase, which derives the wrong identity and loops on "Provided key or passphrase does not match this archive." > > If you configure **only** SSH recipients, ProxSave cannot open its own archives. Always keep at least one `age1...` recipient or a passphrase alongside them. From 019c67bf3bf46566de2ef0f2822d3a2be01a892c Mon Sep 17 00:00:00 2001 From: tis24dev Date: Tue, 4 Aug 2026 14:42:15 +0200 Subject: [PATCH 43/50] docs(cli): close the MD028 blank line between the two scheduling warnings markdownlint flags a bare blank line between consecutive blockquotes because the two render as separate blocks while reading as one. The daemon-versus-cron notice and the crontab-ownership warning are a sequence -- the second qualifies the first -- and the rest of that block already separates its paragraphs with ">", so they are merged the way the file itself does it. Found by running markdownlint over the whole doc set rather than only where a reviewer pointed: this instance was not reported. --- docs/CLI_REFERENCE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/CLI_REFERENCE.md b/docs/CLI_REFERENCE.md index 69475c55..6c39df19 100644 --- a/docs/CLI_REFERENCE.md +++ b/docs/CLI_REFERENCE.md @@ -586,7 +586,7 @@ proxsave --support ## Scheduling with Cron > On fresh installs ProxSave schedules backups through the **resident daemon** (`proxsave-daemon.service`) by default; see [DAEMON.md](DAEMON.md). The daemon runs once daily, so every schedule below (hourly, every 6 hours, weekly, several times a day) requires the daemon-less **cron** engine. Do not add a cron entry while the daemon is active, or the backup runs twice. - +> > **ProxSave owns your crontab, so a hand-written schedule does not survive.** `--install`, `--new-install` and `--daemon-remove` each rewrite it: they delete **every** cron line whose command is named `proxsave` or `proxmox-backup`, not only the one they wrote themselves, and append a single daily entry at `SCHEDULER_TIME`. The deletion happens in both scheduler modes; whether the appended line stays depends on where the run ends. `--daemon-remove` ends on cron, so it keeps it. `--install` and `--new-install` write that line first and then, when the selected (or already configured) mode is `daemon`, drop it again while enabling the unit — so a daemon installation ends with no proxsave cron entry, unless the unit install itself fails and the host stays on cron with the line it just wrote. A custom cadence from this section is therefore silently downgraded to daily by a cron reinstall, and removed outright by a daemon one. `--upgrade` is the exception: it only repoints legacy paths and leaves the schedule alone. > > The practical order is: run `proxsave --daemon-remove` first, which switches to cron, writes the daily line for you, and records the opt-out, **then** edit that line to the cadence you want. Adding a second entry afterwards leaves two, and both will fire. From 90179f9d7dcb04a2335330662e2f9ff71a02b78f Mon Sep 17 00:00:00 2001 From: tis24dev Date: Tue, 4 Aug 2026 14:54:06 +0200 Subject: [PATCH 44/50] fix(cli): report each guard-cleanup outcome as what it was guardApplyFacts collapsed every real run into "removed them" or "hidden under a live mount". Three distinct outcomes came out wrong. A run that finds no guard directory returns an all-zero report, so it read as clean and announced "Removed the restore mount guards. The storage is unlocked." one screen below the engine's own "No guard directory found, nothing to clean up." It removed nothing. GuardsRemaining == -1 is the fail-closed sentinel for a verification reread that failed: the count is unknown. The verdict asserted both that guards remained and why, then told the operator to unmount a datastore -- advice for a diagnosis the run never reached. It now says the count could not be confirmed and asks for a re-run. ImmutablePending covers three causes, commented in the engine as mounted, unresolvable, or the clear failed. Naming only the live mount sends an operator to unmount storage that may never have been involved. Exit codes are untouched: guardApplyClean still decides them and its inputs have not changed. Two existing fixtures described a run that unmounted a guard with no guard directory, which the engine cannot produce; they now set GuardDirPresent. --- cmd/proxsave/cleanup_guards_verdict.go | 35 ++++++++++++++++--- cmd/proxsave/cleanup_guards_verdict_test.go | 38 +++++++++++++++++++-- 2 files changed, 65 insertions(+), 8 deletions(-) diff --git a/cmd/proxsave/cleanup_guards_verdict.go b/cmd/proxsave/cleanup_guards_verdict.go index 52681242..8e74410c 100644 --- a/cmd/proxsave/cleanup_guards_verdict.go +++ b/cmd/proxsave/cleanup_guards_verdict.go @@ -65,15 +65,36 @@ func guardCheckFacts(r orchestrator.GuardCleanupReport) string { return fmt.Sprintf("Found %s locking the storage.", strings.Join(parts, " and ")) } -// guardApplyFacts states the outcome of a real run. The not-clean wording deliberately -// names the usual cause — a guard hidden under a live mount, which the engine refuses -// to unmount — because that is what tells the operator the retry needs the datastore -// offline rather than more privileges. +// guardApplyFacts states the outcome of a real run, keeping the distinctions the engine +// already makes instead of flattening them into clean-or-not. +// +// Three of them matter. A run that found no guard directory removed NOTHING, and saying +// otherwise contradicts the engine's own "nothing to clean up" line one screen earlier. +// GuardsRemaining == -1 is the fail-closed sentinel for a verification reread that +// failed, so the count is unknown and asserting a cause on top of it is two claims the +// run cannot support. And ImmutablePending covers three causes, not one -- the engine +// comments them as mounted, unresolvable, or the clear failed -- so naming only the live +// mount sends an operator to unmount a datastore that was never the problem. func guardApplyFacts(r orchestrator.GuardCleanupReport) string { + if !r.GuardDirPresent { + return "No restore mount guards were present. Nothing to unlock." + } if guardApplyClean(r) { return "Removed the restore mount guards. The storage is unlocked." } - return "Some guards are still in place (hidden under a live mount)." + var parts []string + switch { + case r.GuardsRemaining < 0: + parts = append(parts, "the number of bind mount guards still in place could not be confirmed") + case r.GuardsRemaining > 0: + parts = append(parts, countLabel(r.GuardsRemaining, "bind mount guard")+ + " still in place (hidden under a live mount, or the unmount failed)") + } + if r.ImmutablePending > 0 { + parts = append(parts, countLabel(r.ImmutablePending, "immutable flag")+ + " still set (the target is mounted, unresolvable, or the clear failed)") + } + return "The storage is still locked: " + strings.Join(parts, ", and ") + "." } // logCLIGuardVerdict states the verdict in the CLI's voice: the shared facts plus a @@ -87,6 +108,10 @@ func logCLIGuardVerdict(logger *logging.Logger, r orchestrator.GuardCleanupRepor logger.Info("%s", guardCheckFacts(r)) case guardApplyClean(r): logger.Info("%s", guardApplyFacts(r)) + case r.GuardsRemaining < 0: + // Unknown, not stuck: telling the operator to unmount a datastore would be + // advice for a diagnosis this run never reached. + logger.Warning("%s Run --cleanup-guards again to get a confirmed count.", guardApplyFacts(r)) default: logger.Warning("%s Unmount the datastore and run --cleanup-guards again once it is offline.", guardApplyFacts(r)) } diff --git a/cmd/proxsave/cleanup_guards_verdict_test.go b/cmd/proxsave/cleanup_guards_verdict_test.go index 083ab6b7..27fec94c 100644 --- a/cmd/proxsave/cleanup_guards_verdict_test.go +++ b/cmd/proxsave/cleanup_guards_verdict_test.go @@ -192,14 +192,44 @@ func TestCLIGuardVerdictSaysWhatWasFoundAndWhatToDo(t *testing.T) { }, { name: "real run leaves guards behind", - report: orchestrator.GuardCleanupReport{BindGuards: 1, GuardsRemaining: 1}, + report: orchestrator.GuardCleanupReport{GuardDirPresent: true, BindGuards: 1, GuardsRemaining: 1}, want: []string{"still in place", "hidden under a live mount", "Unmount the datastore", "--cleanup-guards"}, }, { name: "real run unlocks the storage", - report: orchestrator.GuardCleanupReport{BindGuards: 1, Unmounted: 1}, + report: orchestrator.GuardCleanupReport{GuardDirPresent: true, BindGuards: 1, Unmounted: 1}, want: []string{"storage is unlocked"}, }, + { + // The engine returns an all-zero report and logs "nothing to clean up" when + // the guard directory does not exist. Reporting a removal there contradicts + // the line above it and tells an operator their storage was just unlocked by + // a run that touched nothing. + name: "real run with no guard directory removed nothing", + report: orchestrator.GuardCleanupReport{}, + want: []string{"No restore mount guards were present", "Nothing to unlock"}, + notWant: []string{"Removed", "Unmount the datastore"}, + }, + { + // pending counts targets left immutable for THREE reasons (the engine + // comments them as mounted, unresolvable, or the clear failed). Naming only + // the live mount sends the operator to unmount a datastore that may never + // have been involved. + name: "real run leaves an immutable flag pending", + report: orchestrator.GuardCleanupReport{GuardDirPresent: true, ImmutableGuards: 1, ImmutablePending: 1}, + want: []string{"still locked", "1 immutable flag", "unresolvable", "clear failed"}, + notWant: []string{"bind mount guard"}, + }, + { + // GuardsRemaining == -1 is the fail-closed sentinel for a verification reread + // that failed: the count is unknown. Claiming guards are in place AND naming + // their cause is two assertions the run cannot support, and unmount advice is + // guidance for a diagnosis it never reached. + name: "real run cannot confirm what is left", + report: orchestrator.GuardCleanupReport{GuardDirPresent: true, BindGuards: 1, GuardsRemaining: -1}, + want: []string{"could not be confirmed", "again to get a confirmed count"}, + notWant: []string{"hidden under a live mount", "Unmount the datastore"}, + }, } for _, tc := range cases { @@ -234,7 +264,9 @@ func TestCLIGuardVerdictWarnsWhenTheStorageStaysLocked(t *testing.T) { dryRun bool }{ {"guards found by the check", orchestrator.GuardCleanupReport{BindGuards: 1}, true}, - {"guards left by a real run", orchestrator.GuardCleanupReport{BindGuards: 1, GuardsRemaining: 1}, false}, + {"guards left by a real run", orchestrator.GuardCleanupReport{GuardDirPresent: true, BindGuards: 1, GuardsRemaining: 1}, false}, + {"a real run that cannot confirm what is left", orchestrator.GuardCleanupReport{GuardDirPresent: true, BindGuards: 1, GuardsRemaining: -1}, false}, + {"immutable flags left pending by a real run", orchestrator.GuardCleanupReport{GuardDirPresent: true, ImmutableGuards: 1, ImmutablePending: 1}, false}, } { t.Run(tc.name, func(t *testing.T) { buf := &bytes.Buffer{} From 1c43e2a4c1b542c1d3e968694260c5d7714d941e Mon Sep 17 00:00:00 2001 From: tis24dev Date: Tue, 4 Aug 2026 15:00:55 +0200 Subject: [PATCH 45/50] test(guards): search the whole package, and transitively, for a second entry point The structural test that keeps --cleanup-guards from regrowing its error-only wrapper parsed guards_cleanup.go alone and looked for a literal call to cleanupMountGuards. Both narrowings let the exact wrapper it forbids come back unseen: one in any other file of the package was out of scope, and one that delegates to the EXPORTED entry point -- func CleanupMountGuards(...) error { _, err := CleanupMountGuardsReport(...); return err } which is how anyone would naturally write it back -- names cleanupMountGuards nowhere at all. Proven by adding exactly that file: the old test passed, this one fails. The search now parses every non-test file in the package and follows package-local calls transitively, methods included, since an exported method that reaches the engine is a second way in too. The signature check was also half a check. It counted result FIELDS, which is not the number of results -- (a, b T) is one field with two names -- and verified only the first type, so (GuardCleanupReport, string) would have passed. It now counts results and checks that the second is error. --- .../guards_cleanup_entrypoint_test.go | 180 +++++++++++++++--- 1 file changed, 153 insertions(+), 27 deletions(-) diff --git a/internal/orchestrator/guards_cleanup_entrypoint_test.go b/internal/orchestrator/guards_cleanup_entrypoint_test.go index fe64a045..711bf287 100644 --- a/internal/orchestrator/guards_cleanup_entrypoint_test.go +++ b/internal/orchestrator/guards_cleanup_entrypoint_test.go @@ -4,6 +4,7 @@ import ( "go/ast" "go/parser" "go/token" + "os" "strings" "testing" ) @@ -21,51 +22,176 @@ import ( // keeps it deleted: re-adding a convenience wrapper is an easy, well-meant change, and no // behavioural test fails when an unused one appears. This asserts the engine offers exactly // one exported way in and that it hands the caller the report. +// +// The search is package-wide and TRANSITIVE, and both matter. It used to parse +// guards_cleanup.go alone and look for a literal call to cleanupMountGuards, which left the +// easiest way to reintroduce the defect undetected twice over: a wrapper in any other file +// of this package was invisible, and a wrapper that delegates to the EXPORTED entry point +// -- `func X(...) error { r, err := CleanupMountGuardsReport(...); return err }`, the most +// natural way anyone would write it back -- calls cleanupMountGuards nowhere at all. That is +// the exact wrapper this test exists to forbid. func TestGuardCleanupHasOneExportedEntryPoint(t *testing.T) { + const engine = "cleanupMountGuards" + fset := token.NewFileSet() - file, err := parser.ParseFile(fset, "guards_cleanup.go", nil, 0) + entries, err := os.ReadDir(".") if err != nil { - t.Fatalf("parse guards_cleanup.go: %v", err) + t.Fatalf("read package dir: %v", err) } - var exported []string - for _, decl := range file.Decls { - fn, ok := decl.(*ast.FuncDecl) - if !ok || fn.Recv != nil || !fn.Name.IsExported() || fn.Body == nil { + // callees maps a declared name to the package-local names its body calls; decls keeps + // the declaration so an entry point's signature can be checked. Methods are included: + // an exported method that reaches the engine is a second way in just as much as a + // function is. + callees := map[string]map[string]bool{} + decls := map[string]*ast.FuncDecl{} + parsed := 0 + + for _, e := range entries { + name := e.Name() + if e.IsDir() || !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") { continue } - calls := false - ast.Inspect(fn.Body, func(node ast.Node) bool { - call, ok := node.(*ast.CallExpr) - if !ok { - return true - } - if ident, ok := call.Fun.(*ast.Ident); ok && ident.Name == "cleanupMountGuards" { - calls = true + file, err := parser.ParseFile(fset, name, nil, 0) + if err != nil { + t.Fatalf("parse %s: %v", name, err) + } + parsed++ + for _, decl := range file.Decls { + fn, ok := decl.(*ast.FuncDecl) + if !ok || fn.Body == nil { + continue } - return true - }) - if !calls { - continue + key := declKey(fn) + decls[key] = fn + called := map[string]bool{} + ast.Inspect(fn.Body, func(node ast.Node) bool { + call, ok := node.(*ast.CallExpr) + if !ok { + return true + } + switch fun := call.Fun.(type) { + case *ast.Ident: // packageLocalFunc(...) + called[fun.Name] = true + case *ast.SelectorExpr: // o.method(...) / pkg.Func(...) -- keep the selector + called[fun.Sel.Name] = true + } + return true + }) + callees[key] = called } - exported = append(exported, fn.Name.Name) + } - // The one entry point must return the report, not just an error. - if fn.Type.Results == nil || len(fn.Type.Results.List) != 2 { - t.Fatalf("%s must return (GuardCleanupReport, error); a report-less entry point is what let "+ - "--cleanup-guards exit 0 with guards still in place", fn.Name.Name) + if parsed == 0 { + t.Fatal("parsed no production files; the matcher has gone stale") + } + if _, ok := decls[engine]; !ok { + t.Fatalf("%s not found in this package; the matcher has gone stale", engine) + } + + var exported []string + for key, fn := range decls { + if !fn.Name.IsExported() || key == engine { + continue } - first, ok := fn.Type.Results.List[0].Type.(*ast.Ident) - if !ok || first.Name != "GuardCleanupReport" { - t.Fatalf("%s returns %v first, want GuardCleanupReport", fn.Name.Name, fn.Type.Results.List[0].Type) + if !reachesEngine(key, engine, callees, map[string]bool{}) { + continue } + exported = append(exported, key) + assertReportAndError(t, key, fn) } if len(exported) != 1 { + sort := append([]string(nil), exported...) + strSort(sort) t.Fatalf("guard cleanup must have exactly ONE exported entry point, found %d: %s", - len(exported), strings.Join(exported, ", ")) + len(exported), strings.Join(sort, ", ")) } if exported[0] != "CleanupMountGuardsReport" { t.Fatalf("the entry point is %s, want CleanupMountGuardsReport", exported[0]) } } + +// declKey names a declaration: "Recv.Method" for a method, the bare name for a function. +func declKey(fn *ast.FuncDecl) string { + if fn.Recv == nil || len(fn.Recv.List) == 0 { + return fn.Name.Name + } + return recvTypeName(fn.Recv.List[0].Type) + "." + fn.Name.Name +} + +func recvTypeName(expr ast.Expr) string { + switch t := expr.(type) { + case *ast.StarExpr: + return recvTypeName(t.X) + case *ast.Ident: + return t.Name + } + return "?" +} + +// reachesEngine reports whether key's body reaches engine, directly or through any chain of +// package-local calls. seen breaks recursion. +func reachesEngine(key, engine string, callees map[string]map[string]bool, seen map[string]bool) bool { + if seen[key] { + return false + } + seen[key] = true + for callee := range callees[key] { + if callee == engine { + return true + } + // Selector calls are recorded by their final identifier, so a method chain is + // followed by name; an unrelated name simply has no entry here. + for candidate := range callees { + if candidate == callee || strings.HasSuffix(candidate, "."+callee) { + if reachesEngine(candidate, engine, callees, seen) { + return true + } + } + } + } + return false +} + +// assertReportAndError checks the entry point hands back the report AND an error. Counting +// result FIELDS is not the same as counting results -- `(a, b T)` is one field with two +// names -- and checking only the first type would accept (GuardCleanupReport, string). +func assertReportAndError(t *testing.T, name string, fn *ast.FuncDecl) { + t.Helper() + want := "must return (GuardCleanupReport, error); a report-less entry point is what let " + + "--cleanup-guards exit 0 with guards still in place" + + if fn.Type.Results == nil { + t.Fatalf("%s returns nothing and %s", name, want) + } + var types []ast.Expr + for _, field := range fn.Type.Results.List { + n := len(field.Names) + if n == 0 { + n = 1 + } + for i := 0; i < n; i++ { + types = append(types, field.Type) + } + } + if len(types) != 2 { + t.Fatalf("%s returns %d values and %s", name, len(types), want) + } + if id, ok := types[0].(*ast.Ident); !ok || id.Name != "GuardCleanupReport" { + t.Fatalf("%s returns %v first, want GuardCleanupReport", name, types[0]) + } + if id, ok := types[1].(*ast.Ident); !ok || id.Name != "error" { + t.Fatalf("%s returns %v second, want error", name, types[1]) + } +} + +// strSort is an insertion sort, kept local so the failure message is stable without pulling +// sort into a file that otherwise only walks an AST. +func strSort(s []string) { + for i := 1; i < len(s); i++ { + for j := i; j > 0 && s[j] < s[j-1]; j-- { + s[j], s[j-1] = s[j-1], s[j] + } + } +} From 6ec0d6f9d2540fd22b567f5e5bce5e24e3f7771c Mon Sep 17 00:00:00 2001 From: tis24dev Date: Tue, 4 Aug 2026 15:06:02 +0200 Subject: [PATCH 46/50] test(daemon): fail when a new exit code is left unclassified TestExitProvesLockWasTakenClassifiesEveryDocumentedCode checked the codes named in its own two slices, so the rule its doc comment states -- anything added to the accepted column must be traced to a producer that runs after RunPreBackupChecks -- was not enforced by anything. A code added to internal/types/exit_codes.go landed in neither slice, stayed unclassified, and the test went on passing while exitProvesLockWasTaken silently rejected it. On the no-usable-pid branch that means an abandon degrade nothing can lift. Every declared code must now sit in exactly one column. The declared set is read from the source by declaredExitCodes, extracted from TestExitCodesAreDocumented which already did exactly that: a list restated here would drift the same way the documents it checks did. exitCodeInterrupted is classified too. It is not a types constant, but a standalone run that is cancelled hands 130 to the daemon through the same handoff, and a cancelled run proves nothing about the lock. Verified by adding ExitProbeOnly = 99: "appears in 0 of the two columns". --- cmd/proxsave/daemon_abandon_test.go | 61 +++++++++++++++++------ cmd/proxsave/exit_codes_doc_drift_test.go | 58 ++++++++++++--------- 2 files changed, 79 insertions(+), 40 deletions(-) diff --git a/cmd/proxsave/daemon_abandon_test.go b/cmd/proxsave/daemon_abandon_test.go index d4be3f47..98c1af8d 100644 --- a/cmd/proxsave/daemon_abandon_test.go +++ b/cmd/proxsave/daemon_abandon_test.go @@ -1717,26 +1717,55 @@ func TestDaemonFileCleanupCannotHoldTheExit(t *testing.T) { func TestExitProvesLockWasTakenClassifiesEveryDocumentedCode(t *testing.T) { // Accepted: reached the end of a backup, or failed in a phase that only RunGoBackup can // reach -- both strictly after the lock gate passed. - for _, c := range []types.ExitCode{ - types.ExitSuccess, types.ExitGenericError, - types.ExitStorageError, types.ExitVerificationError, types.ExitCollectionError, - types.ExitArchiveError, types.ExitCompressionError, types.ExitEncryptionError, - } { - if !exitProvesLockWasTaken(c.Int()) { - t.Errorf("exit %d is only produced after the lock gate; it must count as proof", c.Int()) - } + accepted := []int{ + types.ExitSuccess.Int(), types.ExitGenericError.Int(), + types.ExitStorageError.Int(), types.ExitVerificationError.Int(), + types.ExitCollectionError.Int(), types.ExitArchiveError.Int(), + types.ExitCompressionError.Int(), types.ExitEncryptionError.Int(), } // Rejected: every code a run can report WITHOUT having reached the lock. ExitConfigError is // the one this list exists for -- it is what an aborted or timed-out encryption prompt now // returns, and accepting it would clear a marker over a live orphan. - for _, c := range []types.ExitCode{ - types.ExitConfigError, types.ExitEnvironmentError, types.ExitBackupError, - types.ExitNetworkError, types.ExitPermissionError, types.ExitDiskSpaceError, - types.ExitPanicError, types.ExitSecurityError, types.ExitBackupSkipped, - types.ExitGuardsPending, - } { - if exitProvesLockWasTaken(c.Int()) { - t.Errorf("exit %d can be reported before the lock gate; it must not count as proof", c.Int()) + rejected := []int{ + types.ExitConfigError.Int(), types.ExitEnvironmentError.Int(), types.ExitBackupError.Int(), + types.ExitNetworkError.Int(), types.ExitPermissionError.Int(), types.ExitDiskSpaceError.Int(), + types.ExitPanicError.Int(), types.ExitSecurityError.Int(), types.ExitBackupSkipped.Int(), + types.ExitGuardsPending.Int(), exitCodeInterrupted, + } + + for _, c := range accepted { + if !exitProvesLockWasTaken(c) { + t.Errorf("exit %d is only produced after the lock gate; it must count as proof", c) + } + } + for _, c := range rejected { + if exitProvesLockWasTaken(c) { + t.Errorf("exit %d can be reported before the lock gate; it must not count as proof", c) + } + } + + // Exhaustiveness. Without this the doc comment on exitProvesLockWasTaken is a promise + // nothing keeps: a code added to internal/types/exit_codes.go lands in neither list + // above, stays unclassified, and this test goes on passing while the function silently + // rejects it -- which on the no-usable-pid branch means a degrade nothing can lift. + // Whoever adds a code has to trace it to its producer and put it in a column. + // + // The declared set is read from the source by declaredExitCodes (see + // exit_codes_doc_drift_test.go), for the same reason that test reads it: a list + // restated here would drift exactly like the one it is checking. + classified := map[int]int{} + for _, c := range accepted { + classified[c]++ + } + for _, c := range rejected { + classified[c]++ + } + declared := declaredExitCodes(t) + declared["exitCodeInterrupted"] = exitCodeInterrupted + for name, code := range declared { + if n := classified[code]; n != 1 { + t.Errorf("%s (%d) appears in %d of the two columns, want exactly 1: trace it to its producer and classify it", + name, code, n) } } } diff --git a/cmd/proxsave/exit_codes_doc_drift_test.go b/cmd/proxsave/exit_codes_doc_drift_test.go index 2d2fb584..a9829a6b 100644 --- a/cmd/proxsave/exit_codes_doc_drift_test.go +++ b/cmd/proxsave/exit_codes_doc_drift_test.go @@ -37,6 +37,39 @@ var exitCodeDocs = []struct { // The constants are read from the source rather than listed here on purpose: a test that // repeats the list drifts in exactly the same way the docs did. func TestExitCodesAreDocumented(t *testing.T) { + codes := declaredExitCodes(t) + // The interrupted code lives in main (128 + SIGINT), not in the types package, but it + // is part of the same published contract. + codes["exitCodeInterrupted"] = exitCodeInterrupted + + for _, doc := range exitCodeDocs { + body, readErr := os.ReadFile(doc.path) + if readErr != nil { + t.Fatalf("read %s: %v", doc.name, readErr) + } + text := string(body) + var missing []string + for name, code := range codes { + // A table row for the code: "| `17` | ... |". Matching the row rather than the + // bare number avoids passing on an unrelated "17" elsewhere in the prose. + row := regexp.MustCompile(`(?m)^\|\s*` + "`" + strconv.Itoa(code) + "`" + `\s*\|`) + if !row.MatchString(text) { + missing = append(missing, fmt.Sprintf("%s (%d)", name, code)) + } + } + if len(missing) > 0 { + t.Errorf("%s has no table row for %d exit code(s): %s", + doc.name, len(missing), strings.Join(missing, ", ")) + } + } +} + +// declaredExitCodes reads the exit-code constants straight out of +// internal/types/exit_codes.go. Reading them from the source rather than restating them +// here is what keeps a list from drifting the way the documents did, so every caller +// inherits that property instead of growing its own copy. +func declaredExitCodes(t *testing.T) map[string]int { + t.Helper() fset := token.NewFileSet() file, err := parser.ParseFile(fset, "../../internal/types/exit_codes.go", nil, 0) if err != nil { @@ -91,30 +124,7 @@ func TestExitCodesAreDocumented(t *testing.T) { if len(codes) == 0 { t.Fatalf("no exit-code constants found; the matcher has gone stale") } - // The interrupted code lives in main (128 + SIGINT), not in the types package, but it - // is part of the same published contract. - codes["exitCodeInterrupted"] = exitCodeInterrupted - - for _, doc := range exitCodeDocs { - body, readErr := os.ReadFile(doc.path) - if readErr != nil { - t.Fatalf("read %s: %v", doc.name, readErr) - } - text := string(body) - var missing []string - for name, code := range codes { - // A table row for the code: "| `17` | ... |". Matching the row rather than the - // bare number avoids passing on an unrelated "17" elsewhere in the prose. - row := regexp.MustCompile(`(?m)^\|\s*` + "`" + strconv.Itoa(code) + "`" + `\s*\|`) - if !row.MatchString(text) { - missing = append(missing, fmt.Sprintf("%s (%d)", name, code)) - } - } - if len(missing) > 0 { - t.Errorf("%s has no table row for %d exit code(s): %s", - doc.name, len(missing), strings.Join(missing, ", ")) - } - } + return codes } // TestExitCodeProseCountsTheBenignCodes pins the sentence a reader actually acts on. From 76b735198d7f2ed5f5a53a257f24f3eefb436332 Mon Sep 17 00:00:00 2001 From: tis24dev Date: Tue, 4 Aug 2026 15:13:16 +0200 Subject: [PATCH 47/50] test(config): neutralise environment overrides before comparing two configs LoadConfigWithBaseDir ends in loadEnvOverrides, so an allowlisted variable set in a developer's shell or a CI job lands on every config loadEnvForTest builds. TestBoolDefaultsMatchTheShippedTemplate then compared two configs that agreed because of the override rather than because the code agreed, and a genuine template-versus-default divergence for that key passed unnoticed -- the exact defect the test was written to catch after PXAR_SCAN_ENABLE shipped one way and compiled the other. Demonstrated on a real divergence (compiled default flipped against the template): clean environment fails, BACKUP_ENABLED=true with this change still fails, and BACKUP_ENABLED=true without it passes. The key list is hoisted to a package-level envOverrideKeys so the helper clears exactly what loadEnvOverrides reads. A copy in the test would drift from the original silently, which is the same failure mode again. t.Setenv(key, "") rather than os.Unsetenv: loadEnvOverrides already skips an empty value, t.Setenv restores the original, and it refuses to run inside a parallel test instead of corrupting a sibling's environment -- this package has parallel tests. --- internal/config/config.go | 85 ++++++++++++++++++---------------- internal/config/config_test.go | 14 ++++++ 2 files changed, 60 insertions(+), 39 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index 420570cb..944d9515 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -406,46 +406,53 @@ func LoadConfigWithBaseDir(configPath, detectedBaseDir string) (*Config, error) return cfg, nil } -// loadEnvOverrides checks for environment variables and overrides config file values -// This allows environment variables to take precedence over file configuration +// envOverrideKeys lists every configuration key an environment variable may override. +// +// Package-level rather than local to loadEnvOverrides because the tests need the same +// list: one that compares two configs loaded from different files has to neutralise these +// first, or an override present in the shell applies to BOTH sides, they agree because of +// it, and a genuine divergence for that key is hidden. A second copy in the test would +// drift away from this one silently, which is the failure mode the comparison exists to +// catch in the first place. +var envOverrideKeys = []string{ + "BACKUP_ENABLED", "DRY_RUN", "DEBUG_LEVEL", "USE_COLOR", "COLORIZE_STEP_LOGS", + "PROFILING_ENABLED", + "COMPRESSION_TYPE", "COMPRESSION_LEVEL", "COMPRESSION_THREADS", "COMPRESSION_MODE", + "ENABLE_DEDUPLICATION", "ENABLE_PREFILTER", "PREFILTER_MAX_FILE_SIZE_MB", + "BACKUP_PATH", "LOG_PATH", "LOCK_PATH", "SECURE_ACCOUNT", + "SECONDARY_ENABLED", "SECONDARY_PATH", "SECONDARY_LOG_PATH", + "CLOUD_ENABLED", "CLOUD_REMOTE", "CLOUD_REMOTE_PATH", "CLOUD_LOG_PATH", + "CLOUD_UPLOAD_MODE", "CLOUD_PARALLEL_MAX_JOBS", "CLOUD_PARALLEL_VERIFICATION", + "CLOUD_WRITE_HEALTHCHECK", + "RCLONE_TIMEOUT_CONNECTION", "RCLONE_TIMEOUT_OPERATION", + "RCLONE_BANDWIDTH_LIMIT", "RCLONE_TRANSFERS", "RCLONE_RETRIES", "RCLONE_VERIFY_METHOD", + "RCLONE_FLAGS", + "CLOUD_BATCH_SIZE", "CLOUD_BATCH_PAUSE", + "MAX_LOCAL_BACKUPS", "MAX_SECONDARY_BACKUPS", "MAX_CLOUD_BACKUPS", + "RETENTION_DAILY", "RETENTION_WEEKLY", "RETENTION_MONTHLY", "RETENTION_YEARLY", + "BUNDLE_ASSOCIATED_FILES", "ENCRYPT_ARCHIVE", "AGE_RECIPIENT", "AGE_RECIPIENT_FILE", + "TELEGRAM_ENABLE", "TELEGRAM_ENABLED", "BOT_TELEGRAM_TYPE", "TELEGRAM_BOT_TOKEN", "TELEGRAM_CHAT_ID", + "EMAIL_ENABLE", "EMAIL_ENABLED", "EMAIL_DELIVERY_METHOD", "EMAIL_FALLBACK_PMF", "EMAIL_FALLBACK_SENDMAIL", + "EMAIL_RECIPIENT", "EMAIL_FROM", + "GOTIFY_ENABLE", "GOTIFY_ENABLED", "GOTIFY_SERVER_URL", "GOTIFY_TOKEN", + "GOTIFY_PRIORITY_SUCCESS", "GOTIFY_PRIORITY_WARNING", "GOTIFY_PRIORITY_FAILURE", + "WEBHOOK_ENABLE", "WEBHOOK_ENABLED", "WEBHOOK_ENDPOINTS", "WEBHOOK_FORMAT", "WEBHOOK_TIMEOUT", + "WEBHOOK_MAX_RETRIES", "WEBHOOK_RETRY_DELAY", + "METRICS_ENABLED", "METRICS_PATH", + "SECURITY_CHECK_ENABLED", "AUTO_UPDATE_HASHES", "AUTO_FIX_PERMISSIONS", + "CONTINUE_ON_SECURITY_ISSUES", "CHECK_NETWORK_SECURITY", "CHECK_FIREWALL", + "CHECK_OPEN_PORTS", "SUSPICIOUS_PORTS", "PORT_WHITELIST", + "SUSPICIOUS_PROCESSES", "SAFE_BRACKET_PROCESSES", "SAFE_KERNEL_PROCESSES", "SAFE_PROCESSES", + "MIN_DISK_SPACE_PRIMARY_GB", "MIN_DISK_SPACE_SECONDARY_GB", "MIN_DISK_SPACE_CLOUD_GB", + "DISABLE_NETWORK_PREFLIGHT", "BACKUP_EXCLUDE_PATTERNS", + "SKIP_PERMISSION_CHECK", "BACKUP_CONFIG_FILE", + "BACKUP_USER", "BACKUP_GROUP", "SET_BACKUP_PERMISSIONS", +} + +// loadEnvOverrides checks for environment variables and overrides config file values. +// This allows environment variables to take precedence over file configuration. func (c *Config) loadEnvOverrides() { - // List of all configuration keys that can be overridden by environment variables - envKeys := []string{ - "BACKUP_ENABLED", "DRY_RUN", "DEBUG_LEVEL", "USE_COLOR", "COLORIZE_STEP_LOGS", - "PROFILING_ENABLED", - "COMPRESSION_TYPE", "COMPRESSION_LEVEL", "COMPRESSION_THREADS", "COMPRESSION_MODE", - "ENABLE_DEDUPLICATION", "ENABLE_PREFILTER", "PREFILTER_MAX_FILE_SIZE_MB", - "BACKUP_PATH", "LOG_PATH", "LOCK_PATH", "SECURE_ACCOUNT", - "SECONDARY_ENABLED", "SECONDARY_PATH", "SECONDARY_LOG_PATH", - "CLOUD_ENABLED", "CLOUD_REMOTE", "CLOUD_REMOTE_PATH", "CLOUD_LOG_PATH", - "CLOUD_UPLOAD_MODE", "CLOUD_PARALLEL_MAX_JOBS", "CLOUD_PARALLEL_VERIFICATION", - "CLOUD_WRITE_HEALTHCHECK", - "RCLONE_TIMEOUT_CONNECTION", "RCLONE_TIMEOUT_OPERATION", - "RCLONE_BANDWIDTH_LIMIT", "RCLONE_TRANSFERS", "RCLONE_RETRIES", "RCLONE_VERIFY_METHOD", - "RCLONE_FLAGS", - "CLOUD_BATCH_SIZE", "CLOUD_BATCH_PAUSE", - "MAX_LOCAL_BACKUPS", "MAX_SECONDARY_BACKUPS", "MAX_CLOUD_BACKUPS", - "RETENTION_DAILY", "RETENTION_WEEKLY", "RETENTION_MONTHLY", "RETENTION_YEARLY", - "BUNDLE_ASSOCIATED_FILES", "ENCRYPT_ARCHIVE", "AGE_RECIPIENT", "AGE_RECIPIENT_FILE", - "TELEGRAM_ENABLE", "TELEGRAM_ENABLED", "BOT_TELEGRAM_TYPE", "TELEGRAM_BOT_TOKEN", "TELEGRAM_CHAT_ID", - "EMAIL_ENABLE", "EMAIL_ENABLED", "EMAIL_DELIVERY_METHOD", "EMAIL_FALLBACK_PMF", "EMAIL_FALLBACK_SENDMAIL", - "EMAIL_RECIPIENT", "EMAIL_FROM", - "GOTIFY_ENABLE", "GOTIFY_ENABLED", "GOTIFY_SERVER_URL", "GOTIFY_TOKEN", - "GOTIFY_PRIORITY_SUCCESS", "GOTIFY_PRIORITY_WARNING", "GOTIFY_PRIORITY_FAILURE", - "WEBHOOK_ENABLE", "WEBHOOK_ENABLED", "WEBHOOK_ENDPOINTS", "WEBHOOK_FORMAT", "WEBHOOK_TIMEOUT", - "WEBHOOK_MAX_RETRIES", "WEBHOOK_RETRY_DELAY", - "METRICS_ENABLED", "METRICS_PATH", - "SECURITY_CHECK_ENABLED", "AUTO_UPDATE_HASHES", "AUTO_FIX_PERMISSIONS", - "CONTINUE_ON_SECURITY_ISSUES", "CHECK_NETWORK_SECURITY", "CHECK_FIREWALL", - "CHECK_OPEN_PORTS", "SUSPICIOUS_PORTS", "PORT_WHITELIST", - "SUSPICIOUS_PROCESSES", "SAFE_BRACKET_PROCESSES", "SAFE_KERNEL_PROCESSES", "SAFE_PROCESSES", - "MIN_DISK_SPACE_PRIMARY_GB", "MIN_DISK_SPACE_SECONDARY_GB", "MIN_DISK_SPACE_CLOUD_GB", - "DISABLE_NETWORK_PREFLIGHT", "BACKUP_EXCLUDE_PATTERNS", - "SKIP_PERMISSION_CHECK", "BACKUP_CONFIG_FILE", - "BACKUP_USER", "BACKUP_GROUP", "SET_BACKUP_PERMISSIONS", - } - - for _, key := range envKeys { + for _, key := range envOverrideKeys { if envValue := os.Getenv(key); envValue != "" { upperKey := strings.ToUpper(key) if canonicalKey, ok := legacyNotificationEnableAliases[upperKey]; ok { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 38a36e11..0033a4b3 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -1632,6 +1632,20 @@ func TestParseEnvFileHandlesExportLines(t *testing.T) { // loader, so both sides of a comparison are read by the code that ships. func loadEnvForTest(t *testing.T, name, content string) *Config { t.Helper() + // Neutralise every environment override first. LoadConfigWithBaseDir ends in + // loadEnvOverrides, so an allowlisted variable set in the developer's shell or in a CI + // job lands on EVERY config this helper builds. A test that compares two of them then + // sees them agree because of the override rather than because the code agrees, and a + // real divergence for that key passes unnoticed -- the exact class of defect + // TestBoolDefaultsMatchTheShippedTemplate exists to catch. + // + // Setting them empty rather than unsetting them is deliberate: loadEnvOverrides skips + // an empty value, t.Setenv restores the original, and unlike os.Unsetenv it refuses to + // run inside a parallel test instead of corrupting a sibling's environment. + for _, key := range envOverrideKeys { + t.Setenv(key, "") + } + path := filepath.Join(t.TempDir(), name) if err := os.WriteFile(path, []byte(content), 0o644); err != nil { t.Fatalf("write %s: %v", name, err) From 8acf3287d82422814a16cb09a7e73ed724b555e9 Mon Sep 17 00:00:00 2001 From: tis24dev Date: Tue, 4 Aug 2026 15:17:09 +0200 Subject: [PATCH 48/50] test(install): let the compiler enforce that setEnvValue is test-only config_helpers.go carried a comment asking production code not to call setEnvValue: the install wizard routes its writes through installer.ApplyInstallData, and a second path that edits a template key by key is how the two drift apart. The function had no production caller, only fixture builders in install_characterization_test.go and helpers_test.go. Moved to helpers_test.go, so the rule is checked rather than requested. Adding a production call site now fails the build with "undefined: setEnvValue" instead of passing review on the strength of a comment nobody has to read. pkg/utils is no longer imported by config_helpers.go. --- cmd/proxsave/config_helpers.go | 10 ---------- cmd/proxsave/helpers_test.go | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/cmd/proxsave/config_helpers.go b/cmd/proxsave/config_helpers.go index b9956505..22114c4c 100644 --- a/cmd/proxsave/config_helpers.go +++ b/cmd/proxsave/config_helpers.go @@ -5,8 +5,6 @@ import ( "os" "path/filepath" "strings" - - "github.com/tis24dev/proxsave/pkg/utils" ) type configStatusLogger interface { @@ -45,14 +43,6 @@ func ensureConfigExists(path string, logger configStatusLogger) error { return fmt.Errorf("configuration file is required to continue") } -// setEnvValue is TEST-ONLY since the install wizard started routing its writes -// through installer.ApplyInstallData: cmd/proxsave/install_characterization_test.go -// and cmd/proxsave/helpers_test.go build fixture templates with it. Production code -// must use installer.SetEnvValueInTemplate / the install engine instead. -func setEnvValue(template, key, value string) string { - return utils.SetEnvValue(template, key, value) -} - func sanitizeEnvValue(value string) string { value = strings.Map(func(r rune) rune { if r == '\n' || r == '\r' || r == '\x00' { diff --git a/cmd/proxsave/helpers_test.go b/cmd/proxsave/helpers_test.go index 2b93cb7b..c45a597c 100644 --- a/cmd/proxsave/helpers_test.go +++ b/cmd/proxsave/helpers_test.go @@ -12,8 +12,22 @@ import ( "github.com/tis24dev/proxsave/internal/config" "github.com/tis24dev/proxsave/internal/input" + "github.com/tis24dev/proxsave/pkg/utils" ) +// setEnvValue builds fixture templates for the tests in this package +// (install_characterization_test.go and the cases below). +// +// It lives in a _test.go file so the COMPILER enforces what used to be a comment in +// config_helpers.go asking production code not to call it. The install wizard routes +// its writes through installer.ApplyInstallData, and a second path that edits a +// template key by key is how the two drift apart; production must use +// installer.SetEnvValueInTemplate or the install engine instead. A note saying so does +// not stop the next call site from appearing -- an undefined symbol does. +func setEnvValue(template, key, value string) string { + return utils.SetEnvValue(template, key, value) +} + // ============================================================ // config_helpers.go tests // ============================================================ From 2c7cc70ee76e832a82d623afcdc93b0e028abfcb Mon Sep 17 00:00:00 2001 From: tis24dev Date: Tue, 4 Aug 2026 15:21:03 +0200 Subject: [PATCH 49/50] docs(whatsnew): announce that PBS datastore scanning is now off by default Reconciling PXAR_SCAN_ENABLE against the shipped template fixed a release that behaved two ways -- a fresh install had datastore scanning off, a config old enough to predate the key had it on -- but it does so by turning that scanning OFF on exactly the configs that had it. Those operators lose that content from their next backup without having changed anything, and nothing told them. The release-notes registry is where that belongs rather than a changelog: it is shown to the operator on upgrade and has a section for steps they must take. The action names backup.env because no wizard, dashboard or CLI path exposes this key -- the registry's rule is to prefer a dashboard path where one exists, and none does. --- internal/whatsnew/registry.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/whatsnew/registry.go b/internal/whatsnew/registry.go index fbd666e4..fa0a3724 100644 --- a/internal/whatsnew/registry.go +++ b/internal/whatsnew/registry.go @@ -61,9 +61,11 @@ var notes = []Note{ "Scheduled backups now run from a resident daemon (replaces cron, reversible), on by default", "Backup monitoring (healthchecks) is on by default and alerts you if a backup stops running", "Host backup mode: back up a Proxmox host from an LXC or HA-LXC appliance", + "PBS datastore file scanning is off by default, including on configs written before the setting existed", }, Actions: []string{ "In the dashboard, open Healthchecks for your monitoring portal address and sign-in details", + "If your backups included PBS datastore file scans, set PXAR_SCAN_ENABLE=true in backup.env to keep them", }, }, } From 4bbeeefe46de79fa8fd24837db70300132f974e5 Mon Sep 17 00:00:00 2001 From: tis24dev Date: Tue, 4 Aug 2026 15:28:35 +0200 Subject: [PATCH 50/50] test(install): track step errors by occurrence, not by identifier name TestEveryTUIInstallStepIsGuarded kept two maps keyed by identifier name, and runInstallTUI reuses `err` across several flowinstall calls. So the three assignments collapsed into one entry -- the count was wrong and only the last position survived -- and a single guarded `err` anywhere in the driver marked every other `err` guarded too. A step whose error never reached an abort decision passed unnoticed, which is the one thing this test exists to catch. Occurrences are now (name, position) pairs, and one counts as guarded only when a decision mentions that name after its assignment and before the next assignment that rebinds it: the window in which the value still belongs to that step. Proven by removing the guards in one window: the old test passed, this one fails with "every install step's error must reach an abort decision; 1 do not". The window is positional, so a guard in a different branch of the same window still counts. That is an approximation, and the comment says so, but a far tighter one than treating the whole function as a single scope. --- cmd/proxsave/install_step_abort_test.go | 55 +++++++++++++++++++++---- 1 file changed, 47 insertions(+), 8 deletions(-) diff --git a/cmd/proxsave/install_step_abort_test.go b/cmd/proxsave/install_step_abort_test.go index 241f2a67..2e4ceb7c 100644 --- a/cmd/proxsave/install_step_abort_test.go +++ b/cmd/proxsave/install_step_abort_test.go @@ -107,9 +107,26 @@ func TestEveryTUIInstallStepIsGuarded(t *testing.T) { t.Fatal("runInstallTUI not found; this test can no longer see the driver it pins") } - // Error identifiers produced by a flow call, and identifiers that reach a decision. - stepErrors := map[string]token.Pos{} - guarded := map[string]bool{} + // Every flowinstall step error, and every identifier reaching a decision, kept as + // (name, position) OCCURRENCES rather than as a set of names. + // + // Names alone are not identifiers here. runInstallTUI reuses `err` for several + // flowinstall calls, so a single map keyed by name collapsed them into one entry -- + // the count was wrong, only the last position survived, and one guarded `err` + // anywhere in the function marked every other `err` guarded too. A step whose error + // never reached a decision passed unnoticed, which is the whole thing this test + // exists to catch. + // + // An occurrence is guarded when a decision mentions that name after the assignment + // and before the NEXT assignment to the same name: the window in which the value + // still belongs to this step. That is positional, so a guard sitting in a different + // branch of the same window still counts -- an approximation, but a strictly tighter + // one than treating the whole function as a single scope. + type occurrence struct { + name string + pos token.Pos + } + var stepErrors, guards []occurrence ast.Inspect(driver, func(node ast.Node) bool { switch n := node.(type) { @@ -131,7 +148,7 @@ func TestEveryTUIInstallStepIsGuarded(t *testing.T) { return true } if ident, ok := n.Lhs[1].(*ast.Ident); ok && ident.Name != "_" { - stepErrors[ident.Name] = n.Pos() + stepErrors = append(stepErrors, occurrence{name: ident.Name, pos: n.Pos()}) } case *ast.CallExpr: // abortInstallOnOptionalStep(ctx, bootstrap, "step", ) or mapUIDeath() @@ -147,7 +164,7 @@ func TestEveryTUIInstallStepIsGuarded(t *testing.T) { } for _, arg := range n.Args { if ident, ok := arg.(*ast.Ident); ok { - guarded[ident.Name] = true + guards = append(guards, occurrence{name: ident.Name, pos: n.Pos()}) } } } @@ -158,9 +175,31 @@ func TestEveryTUIInstallStepIsGuarded(t *testing.T) { t.Fatal("no flowinstall step calls found in runInstallTUI; the matcher has gone stale") } var unguarded []string - for name, pos := range stepErrors { - if !guarded[name] { - unguarded = append(unguarded, fmt.Sprintf("%s (%s)", name, fset.Position(pos))) + for _, step := range stepErrors { + // The window this value owns: from its assignment to the next assignment that + // rebinds the same name, or to the end of the driver. + windowEnd, bounded := token.Pos(0), false + for _, other := range stepErrors { + if other.name != step.name || other.pos <= step.pos { + continue + } + if !bounded || other.pos < windowEnd { + windowEnd, bounded = other.pos, true + } + } + decided := false + for _, g := range guards { + if g.name != step.name || g.pos <= step.pos { + continue + } + if bounded && g.pos >= windowEnd { + continue + } + decided = true + break + } + if !decided { + unguarded = append(unguarded, fmt.Sprintf("%s (%s)", step.name, fset.Position(step.pos))) } } if len(unguarded) > 0 {