diff --git a/.gitignore b/.gitignore index 0bf6e03..0aa9ace 100644 --- a/.gitignore +++ b/.gitignore @@ -23,7 +23,9 @@ graphify-out/ # Local Pi runtime state .atl/ .codegraph/ +.worktrees/ # Calibration worksheets carry private session text — never commit docs/eval/corrections-labeling-*.csv docs/eval/corrections-labeling-*.md +.pi/subagents-debug.log diff --git a/cmd/backscroll/annotate.go b/cmd/backscroll/annotate.go index be36e6d..d888359 100644 --- a/cmd/backscroll/annotate.go +++ b/cmd/backscroll/annotate.go @@ -1,13 +1,13 @@ package main import ( + "context" "fmt" "io" "github.com/spf13/cobra" "github.com/pablontiv/backscroll/internal/config" - "github.com/pablontiv/backscroll/internal/storage" ) func newAnnotateCmd(stdout, stderr io.Writer) *cobra.Command { @@ -20,8 +20,9 @@ func newAnnotateCmd(stdout, stderr io.Writer) *cobra.Command { ) cmd := &cobra.Command{ - Use: "annotate", - Short: "Annotate a message (agent-classification loop write surface)", + Use: "annotate", + Short: "Annotate a message (agent-classification loop write surface)", + SilenceUsage: true, Long: `Annotate a message with a classification label. Validates message existence before writing. Supports both uuid (preferred) and legacy source_path+ordinal fallback. Re-annotating the same (source_path, ordinal, kind) replaces the label.`, @@ -46,7 +47,7 @@ fallback. Re-annotating the same (source_path, ordinal, kind) replaces the label } func runAnnotate(stdout, stderr io.Writer, - uuid, path string, ordinal int, kind, label string) error { + uuid, path string, ordinal int, kind, label string) (retErr error) { // Early flag validation if uuid == "" && (path == "" || ordinal < 0) { @@ -62,11 +63,14 @@ func runAnnotate(stdout, stderr io.Writer, return fmt.Errorf("load config: %w", err) } - db, err := storage.Open(cfg.DatabasePath) + db, diag, err := prepareIndex(context.Background(), cfg, indexMutation, false) + if diag != nil { + return refuseIndex(stdout, stderr, *diag, false, false) + } if err != nil { - return fmt.Errorf("open database: %w", err) + return fmt.Errorf("prepare index: %w", err) } - defer func() { _ = db.Close() }() + defer func() { retErr = closeIndexDB(db, retErr) }() // Upsert annotation if err := db.UpsertAnnotation(uuid, path, ordinal, kind, label); err != nil { diff --git a/cmd/backscroll/compat_diagnostics_test.go b/cmd/backscroll/compat_diagnostics_test.go new file mode 100644 index 0000000..cc79ee5 --- /dev/null +++ b/cmd/backscroll/compat_diagnostics_test.go @@ -0,0 +1,858 @@ +package main + +import ( + "bytes" + "database/sql" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/pablontiv/backscroll/internal/compat" + "github.com/pablontiv/backscroll/internal/storage" +) + +func TestDirectReadRemainsAvailableButClaimsNoIndexFreshness(t *testing.T) { + fixture, err := filepath.Abs(filepath.Join(fixturesDir(), "pi-session.jsonl")) + if err != nil { + t.Fatalf("resolve fixture path: %v", err) + } + dbPath := newUnsupportedIndexedConsumerDB(t) + before := readDBBytes(t, dbPath) + + stdout, stderr, err := runCmd("read", fixture) + if err != nil { + t.Fatalf("read with unsupported index configured failed: %v\nstderr: %s", err, stderr) + } + for _, want := range []string{"pi manifest fixture signal", "pi visible answer"} { + if !strings.Contains(stdout, want) { + t.Fatalf("read output missing decoded fixture content %q:\n%s", want, stdout) + } + } + for _, forbidden := range []string{"usable", "fresh", "current index"} { + if strings.Contains(strings.ToLower(stdout+stderr), forbidden) { + t.Fatalf("direct read made index-freshness claim %q; stdout=%q stderr=%q", forbidden, stdout, stderr) + } + } + if after := readDBBytes(t, dbPath); !bytes.Equal(after, before) { + t.Fatal("direct read mutated unsupported database bytes") + } +} + +func TestStatusUnhealthyIsReadOnly(t *testing.T) { + dbPath := newUnsupportedIndexedConsumerDB(t) + before := snapshotSQLiteFiles(t, dbPath) + + stdout, stderr, err := runCmd("status") + if err == nil { + t.Fatalf("status succeeded on unsupported index; stdout=%q stderr=%q", stdout, stderr) + } + assertDiagnosticText(t, stdout+stderr, compat.CodeUnsupportedLineage, dbPath) + assertSQLiteFilesUnchanged(t, dbPath, before) +} + +func TestValidateUnhealthyIsReadOnly(t *testing.T) { + dbPath := newUnsupportedIndexedConsumerDB(t) + before := snapshotSQLiteFiles(t, dbPath) + + stdout, stderr, err := runCmd("validate") + if err == nil { + t.Fatalf("validate succeeded on unsupported index; stdout=%q stderr=%q", stdout, stderr) + } + assertDiagnosticText(t, stdout+stderr, compat.CodeUnsupportedLineage, dbPath) + assertSQLiteFilesUnchanged(t, dbPath, before) +} + +func TestValidateTextReportsSemanticRecoveryDiagnosticsReadOnly(t *testing.T) { + cases := []struct { + name string + setup func(t *testing.T) string + wantCode compat.Code + }{ + { + name: "recovery conflict", + setup: newRecoveryConflictDiagnosticDB, + wantCode: compat.CodeRecoveryConflict, + }, + { + name: "uninterpretable row", + setup: newUninterpretableRowDiagnosticDB, + wantCode: compat.CodeUninterpretableRow, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + dbPath := tc.setup(t) + before := snapshotSQLiteFiles(t, dbPath) + + stdout, stderr, err := runCmd("validate") + if err == nil { + t.Fatalf("plain validate succeeded; want %s diagnostic; stdout=%q stderr=%q", tc.wantCode, stdout, stderr) + } + combined := stdout + stderr + assertDiagnosticText(t, combined, tc.wantCode, dbPath) + wantContinuation := fmt.Sprintf("continuation: recover --from %s --dry-run\n", dbPath) + if !strings.Contains(combined, wantContinuation) { + t.Fatalf("plain validate continuation mismatch: want exact line %q in %q", wantContinuation, combined) + } + assertSQLiteFilesUnchanged(t, dbPath, before) + }) + } +} + +func TestValidateTextReportsMultipleSemanticRecoveryDiagnosticsReadOnly(t *testing.T) { + dbPath := newMixedRecoveryDiagnosticDB(t) + before := snapshotSQLiteFiles(t, dbPath) + + stdout, stderr, err := runCmd("validate") + if err == nil { + t.Fatalf("plain validate succeeded; want multiple recovery diagnostics; stdout=%q stderr=%q", stdout, stderr) + } + combined := stdout + stderr + for _, wantCode := range []compat.Code{compat.CodeRecoveryConflict, compat.CodeUninterpretableRow} { + assertDiagnosticText(t, combined, wantCode, dbPath) + } + if got := strings.Count(combined, "continuation: recover --from "+dbPath+" --dry-run"); got < 2 { + t.Fatalf("plain validate continuation count = %d, want at least 2 in %q", got, combined) + } + assertSQLiteFilesUnchanged(t, dbPath, before) +} + +func TestStatusHealthyIndexIsReadOnlyInTextAndJSON(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "backscroll.db") + db, err := storage.Open(dbPath) + if err != nil { + t.Fatalf("create current index: %v", err) + } + if err := db.SyncFiles([]storage.IndexedFile{{ + SourcePath: "/sessions/status.jsonl", + Source: "session", + Hash: "status-hash", + Project: "project", + Messages: []storage.IndexedMessage{{ + Ordinal: 0, Role: "user", Text: "status visible sentinel", UUID: "11111111-1111-4111-8111-111111111111", + Timestamp: "2026-08-18T00:00:00Z", ContentType: "text", ExtractionVersion: storage.CurrentExtractionVersion, + }}, + }}); err != nil { + _ = db.Close() + t.Fatalf("seed current index: %v", err) + } + if err := db.Close(); err != nil { + t.Fatalf("close current index: %v", err) + } + setIndexPolicyEnv(t, dbPath, t.TempDir()) + before := snapshotSQLiteFiles(t, dbPath) + + stdout, stderr, err := runCmd("status") + if err != nil { + t.Fatalf("status healthy index failed: %v stdout=%q stderr=%q", err, stdout, stderr) + } + if stderr != "" || !strings.Contains(stdout, "Files indexed: 1") || !strings.Contains(stdout, "Messages indexed: 1") { + t.Fatalf("status healthy index text output stdout=%q stderr=%q", stdout, stderr) + } + assertSQLiteFilesUnchanged(t, dbPath, before) + + stdout, stderr, err = runCmd("status", "--json") + if err != nil { + t.Fatalf("status --json healthy index failed: %v stdout=%q stderr=%q", err, stdout, stderr) + } + if stderr != "" { + t.Fatalf("status --json healthy index wrote stderr: %q", stderr) + } + var payload struct { + Database struct { + Exists bool `json:"exists"` + Size int `json:"size"` + } `json:"database"` + Index struct { + Usable bool `json:"usable"` + TotalFiles int `json:"total_files"` + TotalMessages int `json:"total_messages"` + } `json:"index"` + } + if err := json.Unmarshal([]byte(stdout), &payload); err != nil { + t.Fatalf("status --json healthy index emitted invalid JSON %q: %v", stdout, err) + } + if !payload.Database.Exists || payload.Database.Size == 0 || !payload.Index.Usable || payload.Index.TotalFiles != 1 || payload.Index.TotalMessages != 1 { + t.Fatalf("status --json healthy index payload = %+v, want existing usable one-row index", payload) + } + assertSQLiteFilesUnchanged(t, dbPath, before) +} + +func TestAnnotatePathOrdinalFallbackPersistsThroughCobra(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "backscroll.db") + db, err := storage.Open(dbPath) + if err != nil { + t.Fatalf("create current index: %v", err) + } + if err := db.SyncFiles([]storage.IndexedFile{{ + SourcePath: "/sessions/fallback.jsonl", + Source: "session", + Hash: "annotate-hash", + Project: "project", + Messages: []storage.IndexedMessage{{ + Ordinal: 3, Role: "assistant", Text: "annotation fallback sentinel", + Timestamp: "2026-08-18T00:00:00Z", ContentType: "text", ExtractionVersion: storage.CurrentExtractionVersion, + }}, + }}); err != nil { + _ = db.Close() + t.Fatalf("seed current index: %v", err) + } + if err := db.Close(); err != nil { + t.Fatalf("close current index: %v", err) + } + setIndexPolicyEnv(t, dbPath, t.TempDir()) + + stdout, stderr, err := runCmd("annotate", "--path", "/sessions/fallback.jsonl", "--ordinal", "3", "--kind", "correction", "--label", "needs-review") + if err != nil { + t.Fatalf("annotate path/ordinal fallback failed: %v stdout=%q stderr=%q", err, stdout, stderr) + } + if stderr != "" || !strings.Contains(stdout, "Annotated /sessions/fallback.jsonl:3 as correction=needs-review") { + t.Fatalf("annotate path/ordinal fallback output stdout=%q stderr=%q", stdout, stderr) + } + + inspect, err := storage.OpenReadOnly(dbPath) + if err != nil { + t.Fatalf("open annotated index read-only: %v", err) + } + defer func() { _ = inspect.Close() }() + var label, source string + if err := inspect.DB().QueryRow(`SELECT label, source FROM annotations WHERE source_path = ? AND ordinal = ? AND kind = ?`, "/sessions/fallback.jsonl", 3, "correction").Scan(&label, &source); err != nil { + t.Fatalf("query persisted annotation: %v", err) + } + if label != "needs-review" || source != "agent" { + t.Fatalf("persisted annotation label/source = %q/%q", label, source) + } +} + +func TestValidateHealthyIndexIsReadOnlyInTextAndJSON(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "backscroll.db") + db, err := storage.Open(dbPath) + if err != nil { + t.Fatalf("create current index: %v", err) + } + if err := db.Close(); err != nil { + t.Fatalf("close current index: %v", err) + } + setIndexPolicyEnv(t, dbPath, t.TempDir()) + before := snapshotSQLiteFiles(t, dbPath) + + stdout, stderr, err := runCmd("validate") + if err != nil { + t.Fatalf("validate healthy index failed: %v stdout=%q stderr=%q", err, stdout, stderr) + } + if stderr != "" || !strings.Contains(stdout, "Index validation passed") { + t.Fatalf("validate healthy index text output stdout=%q stderr=%q", stdout, stderr) + } + assertSQLiteFilesUnchanged(t, dbPath, before) + + stdout, stderr, err = runCmd("validate", "--json") + if err != nil { + t.Fatalf("validate --json healthy index failed: %v stdout=%q stderr=%q", err, stdout, stderr) + } + if stderr != "" { + t.Fatalf("validate --json healthy index wrote stderr: %q", stderr) + } + var payload struct { + Valid bool `json:"valid"` + DatabaseExists bool `json:"database_exists"` + } + if err := json.Unmarshal([]byte(stdout), &payload); err != nil { + t.Fatalf("validate --json healthy index emitted invalid JSON %q: %v", stdout, err) + } + if !payload.Valid || !payload.DatabaseExists { + t.Fatalf("validate --json healthy index payload = %+v, want valid existing database", payload) + } + assertSQLiteFilesUnchanged(t, dbPath, before) +} + +func TestConfigAndStatusDeclarativeInputsVisibleWithoutCreatingIndex(t *testing.T) { + dir := t.TempDir() + dbPath := filepath.Join(dir, "backscroll.db") + cfgDir := filepath.Join(dir, "config") + presetDir := filepath.Join(fixturesDir(), "claude-preset") + setupInputsPreset(t, cfgDir, filepath.Join(presetDir, "projects")) + t.Setenv("HOME", filepath.Join(dir, "home")) + t.Setenv("BACKSCROLL_CONFIG_DIR", cfgDir) + t.Setenv("BACKSCROLL_DATABASE_PATH", dbPath) + + stdout, stderr, err := runCmd("config") + if err != nil { + t.Fatalf("config text with declarative input failed: %v stdout=%q stderr=%q", err, stdout, stderr) + } + for _, want := range []string{"Backscroll Configuration", "Inputs (declarative)", "id: claude", "format: claude"} { + if !strings.Contains(stdout, want) { + t.Fatalf("config text missing %q in:\n%s", want, stdout) + } + } + if stderr != "" { + t.Fatalf("config text wrote stderr: %q", stderr) + } + + stdout, stderr, err = runCmd("config", "--json") + if err != nil { + t.Fatalf("config --json with declarative input failed: %v stdout=%q stderr=%q", err, stdout, stderr) + } + if stderr != "" { + t.Fatalf("config --json wrote stderr: %q", stderr) + } + var payload struct { + Inputs struct { + Mode string `json:"mode"` + Count int `json:"count"` + Manifest []struct { + ID string `json:"id"` + Format string `json:"format"` + Include []string `json:"include"` + } `json:"manifest"` + } `json:"inputs"` + } + if err := json.Unmarshal([]byte(stdout), &payload); err != nil { + t.Fatalf("config --json emitted invalid JSON %q: %v", stdout, err) + } + if payload.Inputs.Mode != "declarative" || payload.Inputs.Count != 1 || len(payload.Inputs.Manifest) != 1 || payload.Inputs.Manifest[0].ID != "claude" || payload.Inputs.Manifest[0].Format != "claude" { + t.Fatalf("config --json declarative payload = %+v", payload.Inputs) + } + + stdout, stderr, err = runCmd("status") + if err != nil { + t.Fatalf("status text with declarative input failed: %v stdout=%q stderr=%q", err, stdout, stderr) + } + for _, want := range []string{"Index: Not yet created", "Inputs: 1 active (declarative)", "- claude"} { + if !strings.Contains(stdout, want) { + t.Fatalf("status text missing %q in:\n%s", want, stdout) + } + } + if stderr != "" { + t.Fatalf("status text wrote stderr: %q", stderr) + } + if _, err := os.Stat(dbPath); !os.IsNotExist(err) { + t.Fatalf("config/status created database: %v", err) + } +} + +func TestStatusAndValidateMissingIndexAreReadOnlyMachineDiagnostics(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "missing", "backscroll.db") + setIndexPolicyEnv(t, dbPath, t.TempDir()) + + stdout, stderr, err := runCmd("validate") + if err != nil { + t.Fatalf("validate missing index failed: %v stdout=%q stderr=%q", err, stdout, stderr) + } + if stderr != "" || !strings.Contains(stdout, "database not found") { + t.Fatalf("validate missing index text output stdout=%q stderr=%q", stdout, stderr) + } + if _, err := os.Stat(dbPath); !os.IsNotExist(err) { + t.Fatalf("validate created or touched missing database: %v", err) + } + + stdout, stderr, err = runCmd("status") + if err != nil { + t.Fatalf("status missing index failed: %v stdout=%q stderr=%q", err, stdout, stderr) + } + if stderr != "" || !strings.Contains(stdout, "Index: Not yet created") || !strings.Contains(stdout, "Messages indexed: 0") { + t.Fatalf("status missing index text output stdout=%q stderr=%q", stdout, stderr) + } + if _, err := os.Stat(dbPath); !os.IsNotExist(err) { + t.Fatalf("status created or touched missing database: %v", err) + } + + stdout, stderr, err = runCmd("validate", "--json") + if err != nil { + t.Fatalf("validate --json missing index failed: %v stdout=%q stderr=%q", err, stdout, stderr) + } + if stderr != "" { + t.Fatalf("validate --json missing index wrote stderr: %q", stderr) + } + var validatePayload struct { + Valid bool `json:"valid"` + DatabaseExists bool `json:"database_exists"` + } + if err := json.Unmarshal([]byte(stdout), &validatePayload); err != nil { + t.Fatalf("validate --json missing index emitted invalid JSON %q: %v", stdout, err) + } + if !validatePayload.Valid || validatePayload.DatabaseExists { + t.Fatalf("validate --json missing index payload = %+v, want valid without database", validatePayload) + } + if _, err := os.Stat(dbPath); !os.IsNotExist(err) { + t.Fatalf("validate --json created or touched missing database: %v", err) + } + + stdout, stderr, err = runCmd("status", "--json") + if err != nil { + t.Fatalf("status --json missing index failed: %v stdout=%q stderr=%q", err, stdout, stderr) + } + if stderr != "" { + t.Fatalf("status --json missing index wrote stderr: %q", stderr) + } + var statusPayload struct { + Database struct { + Exists bool `json:"exists"` + Size int `json:"size"` + } `json:"database"` + Index struct { + Usable bool `json:"usable"` + TotalFiles int `json:"total_files"` + TotalMessages int `json:"total_messages"` + } `json:"index"` + } + if err := json.Unmarshal([]byte(stdout), &statusPayload); err != nil { + t.Fatalf("status --json missing index emitted invalid JSON %q: %v", stdout, err) + } + if statusPayload.Database.Exists || statusPayload.Database.Size != 0 || statusPayload.Index.Usable || statusPayload.Index.TotalFiles != 0 || statusPayload.Index.TotalMessages != 0 { + t.Fatalf("status --json missing index payload = %+v, want no index and zero counts", statusPayload) + } + if _, err := os.Stat(dbPath); !os.IsNotExist(err) { + t.Fatalf("status --json created or touched missing database: %v", err) + } +} + +func TestValidateCurrentIndexIntegrityFailureIsGenericAndReadOnly(t *testing.T) { + dbPath := newOrphanedCurrentDiagnosticDB(t) + before := snapshotSQLiteFiles(t, dbPath) + + stdout, stderr, err := runCmd("validate", "--json") + if err == nil { + t.Fatalf("validate --json orphaned index succeeded; stdout=%q stderr=%q", stdout, stderr) + } + if stderr != "" { + t.Fatalf("validate --json orphaned index wrote stderr: %q", stderr) + } + var payload struct { + Valid bool `json:"valid"` + Error string `json:"error"` + } + if err := json.Unmarshal([]byte(stdout), &payload); err != nil { + t.Fatalf("validate --json orphaned index emitted invalid JSON %q: %v", stdout, err) + } + if payload.Valid || !strings.Contains(payload.Error, "orphaned search_items") { + t.Fatalf("validate --json orphaned index payload = %+v, want generic validation failure", payload) + } + assertSQLiteFilesUnchanged(t, dbPath, before) + + stdout, stderr, err = runCmd("validate") + if err == nil { + t.Fatalf("validate orphaned index succeeded; stdout=%q stderr=%q", stdout, stderr) + } + combined := strings.ToLower(stdout + stderr) + if !strings.Contains(combined, "validation failed") || !strings.Contains(combined, "orphaned search_items") { + t.Fatalf("validate orphaned index output = stdout=%q stderr=%q, want generic validation failure", stdout, stderr) + } + for _, forbidden := range []string{"diagnostic:", "continuation:", "recover --from"} { + if strings.Contains(combined, forbidden) { + t.Fatalf("validate orphaned index emitted recovery diagnostic %q in stdout=%q stderr=%q", forbidden, stdout, stderr) + } + } + assertSQLiteFilesUnchanged(t, dbPath, before) +} + +func TestSearchRobotDiagnosticIsMachineReadableAndPreservesIndexBytes(t *testing.T) { + dbPath := newUnsupportedIndexedConsumerDB(t) + before := readDBBytes(t, dbPath) + + stdout, stderr, err := runCmd("search", "sentinel", "--robot", "--indexed-only") + if err == nil { + t.Fatalf("search --robot unsupported index succeeded; stdout=%q stderr=%q", stdout, stderr) + } + if stderr != "" { + t.Fatalf("search --robot diagnostic wrote stderr: %q", stderr) + } + for _, want := range []string{ + "diagnostic_code=" + string(compat.CodeUnsupportedLineage), + "diagnostic_summary=", + "diagnostic_continuation_argv=recover --from " + dbPath + " --dry-run", + } { + if !strings.Contains(stdout, want) { + t.Fatalf("search --robot diagnostic missing %q in %q", want, stdout) + } + } + if after := readDBBytes(t, dbPath); !bytes.Equal(after, before) { + t.Fatal("search --robot diagnostic mutated unsupported database bytes") + } +} + +func TestLiveWALDiagnosticDoesNotClaimMigrationFailureOrRecovery(t *testing.T) { + dbPath, closeWriter := newLiveWALDiagnosticDB(t) + defer closeWriter() + before := snapshotSQLiteFiles(t, dbPath) + + for _, argv := range [][]string{{"status", "--json"}, {"validate", "--json"}} { + t.Run(strings.Join(argv, " "), func(t *testing.T) { + got := runJSONDiagnosticAllowNoContinuation(t, argv, compat.CodeIndexStale) + if len(got.Continuation) != 0 { + t.Fatalf("%v continuation=%v, want none", argv, got.Continuation) + } + summary := strings.ToLower(got.Summary) + for _, want := range []string{"cannot be inspected without side effects", "wal", "uncheckpointed frames"} { + if !strings.Contains(summary, want) { + t.Fatalf("%v summary missing %q: %q", argv, want, got.Summary) + } + } + for _, forbidden := range []string{"migration_failed", "recover --from", "--dry-run"} { + if strings.Contains(strings.ToLower(got.Code+" "+got.Summary+" "+strings.Join(got.Continuation, " ")), forbidden) { + t.Fatalf("%v emitted false recovery/migration guidance %q in %+v", argv, forbidden, got) + } + } + assertSQLiteFilesUnchanged(t, dbPath, before) + }) + } + + t.Run("status text", func(t *testing.T) { + stdout, stderr, err := runCmd("status") + if err == nil { + t.Fatalf("status text succeeded with live WAL; stdout=%q stderr=%q", stdout, stderr) + } + out := strings.ToLower(stdout + stderr) + for _, want := range []string{"diagnostic: " + string(compat.CodeIndexStale), "wal", "uncheckpointed frames"} { + if !strings.Contains(out, strings.ToLower(want)) { + t.Fatalf("status text live WAL diagnostic missing %q: %q", want, stderr) + } + } + for _, forbidden := range []string{"continuation:", "migration_failed", "recover --from", "--dry-run"} { + if strings.Contains(out, forbidden) { + t.Fatalf("status text emitted false recovery/migration guidance %q in %q", forbidden, stderr) + } + } + assertSQLiteFilesUnchanged(t, dbPath, before) + }) +} + +func TestBlockingDiagnosticsHaveExecutableContinuations(t *testing.T) { + cases := []struct { + name string + setup func(t *testing.T) string + argv []string + wantCode compat.Code + }{ + { + name: "unsupported lineage", + setup: func(t *testing.T) string { + return newUnsupportedIndexedConsumerDB(t) + }, + argv: []string{"status", "--json"}, + wantCode: compat.CodeUnsupportedLineage, + }, + { + name: "migration failure", + setup: func(t *testing.T) string { + return newMigrationFailureDiagnosticDB(t) + }, + argv: []string{"search", "sentinel", "--json"}, + wantCode: compat.CodeMigrationFailed, + }, + { + name: "stale sync", + setup: func(t *testing.T) string { + return newStaleSyncDiagnosticDB(t) + }, + argv: []string{"search", "sentinel", "--json"}, + wantCode: compat.CodeIndexStale, + }, + { + name: "recovery conflict", + setup: func(t *testing.T) string { + return newRecoveryConflictDiagnosticDB(t) + }, + argv: []string{"validate", "--json"}, + wantCode: compat.CodeRecoveryConflict, + }, + { + name: "uninterpretable row", + setup: func(t *testing.T) string { + return newUninterpretableRowDiagnosticDB(t) + }, + argv: []string{"validate", "--json"}, + wantCode: compat.CodeUninterpretableRow, + }, + } + + ran := 0 + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ran++ + dbPath := tc.setup(t) + diagnostic := runJSONDiagnostic(t, tc.argv, tc.wantCode) + want := []string{"recover", "--from", dbPath, "--dry-run"} + if !equalStrings(diagnostic.Continuation, want) { + t.Fatalf("continuation=%v, want %v", diagnostic.Continuation, want) + } + executeContinuationThroughCobra(t, diagnostic.Continuation) + }) + } + if ran != len(cases) { + t.Fatalf("ran %d cases, want %d", ran, len(cases)) + } +} + +type jsonDiagnosticPayload struct { + Code string `json:"code"` + Summary string `json:"summary"` + Continuation []string `json:"continuation_argv"` +} + +func runJSONDiagnostic(t *testing.T, argv []string, wantCode compat.Code) jsonDiagnosticPayload { + t.Helper() + got := runJSONDiagnosticAllowNoContinuation(t, argv, wantCode) + if len(got.Continuation) == 0 { + t.Fatalf("%v emitted empty continuation in diagnostic %+v", argv, got) + } + return got +} + +func runJSONDiagnosticAllowNoContinuation(t *testing.T, argv []string, wantCode compat.Code) jsonDiagnosticPayload { + t.Helper() + var stdout, stderr bytes.Buffer + err := run(&stdout, &stderr, argv) + if err == nil { + t.Fatalf("%v succeeded; stdout=%q stderr=%q", argv, stdout.String(), stderr.String()) + } + if stderr.Len() != 0 { + t.Fatalf("%v JSON diagnostic wrote stderr: %q", argv, stderr.String()) + } + var got jsonDiagnosticPayload + if unmarshalErr := json.Unmarshal(stdout.Bytes(), &got); unmarshalErr != nil { + t.Fatalf("%v emitted invalid JSON diagnostic %q: %v", argv, stdout.String(), unmarshalErr) + } + if got.Code != string(wantCode) { + t.Fatalf("%v code=%q summary=%q, want %q", argv, got.Code, got.Summary, wantCode) + } + return got +} + +func executeContinuationThroughCobra(t *testing.T, argv []string) { + t.Helper() + var stdout, stderr bytes.Buffer + root := buildRootCmd(&stdout, &stderr) + root.SetArgs(argv) + err := root.Execute() + if err != nil && isCobraResolutionError(err) { + t.Fatalf("continuation %v did not resolve through Cobra: %v\nstdout=%q stderr=%q", argv, err, stdout.String(), stderr.String()) + } +} + +func isCobraResolutionError(err error) bool { + if err == nil { + return false + } + msg := err.Error() + return strings.Contains(msg, "unknown command") || strings.Contains(msg, "unknown flag") || strings.Contains(msg, "required flag") || strings.Contains(msg, "accepts") +} + +func assertDiagnosticText(t *testing.T, text string, code compat.Code, dbPath string) { + t.Helper() + for _, want := range []string{string(code), "recover", "--from", dbPath, "--dry-run"} { + if !strings.Contains(text, want) { + t.Fatalf("diagnostic text missing %q: %q", want, text) + } + } +} + +func newMigrationFailureDiagnosticDB(t *testing.T) string { + t.Helper() + dbPath := newFixtureIndexDB(t, "v8.sql") + raw, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatalf("open fixture: %v", err) + } + _, err = raw.Exec(` + INSERT INTO tool_events (message_uuid, source_path, ordinal, tool_name, command_head, is_error, extraction_version) + VALUES ('dup-tool-uuid', '/dup-a.jsonl', 1, 'Bash', 'echo one', 0, 8), + ('dup-tool-uuid', '/dup-b.jsonl', 2, 'Bash', 'echo two', 0, 8) + `) + closeErr := raw.Close() + if err != nil { + t.Fatalf("seed duplicate tool events: %v", err) + } + if closeErr != nil { + t.Fatalf("close fixture: %v", closeErr) + } + setIndexPolicyEnv(t, dbPath, t.TempDir()) + return dbPath +} + +func newStaleSyncDiagnosticDB(t *testing.T) string { + t.Helper() + dbPath := newSupportedIndexedConsumerDB(t) + root := filepath.Join(t.TempDir(), "inputs-root") + if err := os.MkdirAll(root, 0o755); err != nil { + t.Fatalf("mkdir input root: %v", err) + } + setIndexPolicyEnv(t, dbPath, t.TempDir()) + writeInputManifest(t, root, "claude", root, []string{"["}, nil) + writeFile(t, filepath.Join(root, "session.jsonl"), `{"type":"message","message":{"role":"user","content":"fresh"}}`+"\n") + return dbPath +} + +func newLiveWALDiagnosticDB(t *testing.T) (string, func()) { + t.Helper() + dbPath, db := newEmptyCurrentDiagnosticDB(t) + if err := db.SyncFiles([]storage.IndexedFile{{ + SourcePath: "/live-wal/session.jsonl", + Source: "session", + Hash: "live-wal-hash", + Project: "project", + Messages: []storage.IndexedMessage{{ + Ordinal: 0, + Role: "user", + Text: "live WAL committed diagnostic row", + UUID: "44444444-4444-4444-8444-444444444444", + Timestamp: "2026-08-18T00:00:00Z", + ContentType: "text", + }}, + }}); err != nil { + _ = db.Close() + t.Fatalf("seed live WAL diagnostic db: %v", err) + } + wal, err := os.Stat(dbPath + "-wal") + if err != nil { + _ = db.Close() + t.Fatalf("stat live WAL diagnostic sidecar: %v", err) + } + if wal.Size() == 0 { + _ = db.Close() + t.Fatal("live WAL diagnostic sidecar is empty; fixture did not keep committed WAL frames") + } + setIndexPolicyEnv(t, dbPath, t.TempDir()) + return dbPath, func() { _ = db.Close() } +} + +func newRecoveryConflictDiagnosticDB(t *testing.T) string { + t.Helper() + dbPath, db := newEmptyCurrentDiagnosticDB(t) + _, err := db.DB().Exec(` + INSERT INTO search_items (source, source_path, ordinal, role, text, timestamp, project, content_type) + VALUES ('session', '/conflict/session.jsonl', 7, 'user', 'first conflicting payload', '2026-08-18T00:00:00Z', 'project', 'text'), + ('session', '/conflict/session.jsonl', 7, 'assistant', 'second conflicting payload', '2026-08-18T00:00:01Z', 'project', 'text') + `) + if err != nil { + _ = db.Close() + t.Fatalf("seed recovery conflict: %v", err) + } + if err := closeSeedDB(t, db); err != nil { + t.Fatalf("close conflict db: %v", err) + } + setIndexPolicyEnv(t, dbPath, t.TempDir()) + return dbPath +} + +func newUninterpretableRowDiagnosticDB(t *testing.T) string { + t.Helper() + dbPath, db := newEmptyCurrentDiagnosticDB(t) + _, err := db.DB().Exec(` + INSERT INTO search_items (source, source_path, ordinal, role, text, timestamp, project, content_type) + VALUES ('session', '', -1, 'user', 'missing identity payload', '2026-08-18T00:00:00Z', 'project', 'text') + `) + if err != nil { + _ = db.Close() + t.Fatalf("seed uninterpretable row: %v", err) + } + if err := closeSeedDB(t, db); err != nil { + t.Fatalf("close uninterpretable db: %v", err) + } + setIndexPolicyEnv(t, dbPath, t.TempDir()) + return dbPath +} + +func newMixedRecoveryDiagnosticDB(t *testing.T) string { + t.Helper() + dbPath, db := newEmptyCurrentDiagnosticDB(t) + _, err := db.DB().Exec(` + INSERT INTO search_items (source, source_path, ordinal, role, text, timestamp, project, content_type) + VALUES ('session', '/conflict/session.jsonl', 7, 'user', 'first conflicting payload', '2026-08-18T00:00:00Z', 'project', 'text'), + ('session', '/conflict/session.jsonl', 7, 'assistant', 'second conflicting payload', '2026-08-18T00:00:01Z', 'project', 'text'), + ('session', '', -1, 'user', 'missing identity payload', '2026-08-18T00:00:02Z', 'project', 'text') + `) + if err != nil { + _ = db.Close() + t.Fatalf("seed mixed recovery diagnostics: %v", err) + } + if err := closeSeedDB(t, db); err != nil { + t.Fatalf("close mixed diagnostics db: %v", err) + } + setIndexPolicyEnv(t, dbPath, t.TempDir()) + return dbPath +} + +func newOrphanedCurrentDiagnosticDB(t *testing.T) string { + t.Helper() + dbPath, db := newEmptyCurrentDiagnosticDB(t) + _, err := db.DB().Exec(` + INSERT INTO search_items (source, source_path, ordinal, role, text, timestamp, uuid, project, content_type) + VALUES ('session', '/orphaned/session.jsonl', 0, 'user', 'orphaned validation sentinel', '2026-08-18T00:00:00Z', '11111111-1111-4111-8111-111111111111', 'project', 'text') + `) + if err != nil { + _ = db.Close() + t.Fatalf("seed orphaned current db: %v", err) + } + if err := closeSeedDB(t, db); err != nil { + t.Fatalf("close orphaned db: %v", err) + } + setIndexPolicyEnv(t, dbPath, t.TempDir()) + return dbPath +} + +func newEmptyCurrentDiagnosticDB(t *testing.T) (string, *storage.Database) { + t.Helper() + dbPath := filepath.Join(t.TempDir(), "index.db") + db, err := storage.Open(dbPath) + if err != nil { + t.Fatalf("open current diagnostic db: %v", err) + } + resolved, err := filepath.EvalSymlinks(dbPath) + if err != nil { + _ = db.Close() + t.Fatalf("resolve current diagnostic db: %v", err) + } + return resolved, db +} + +func closeSeedDB(t *testing.T, db *storage.Database) error { + t.Helper() + if _, err := db.DB().Exec(`PRAGMA wal_checkpoint(TRUNCATE)`); err != nil { + _ = db.Close() + return fmt.Errorf("checkpoint: %w", err) + } + return db.Close() +} + +func snapshotSQLiteFiles(t *testing.T, dbPath string) map[string][]byte { + t.Helper() + snapshot := make(map[string][]byte) + for _, path := range []string{dbPath, dbPath + "-wal", dbPath + "-shm", dbPath + "-journal"} { + data, err := os.ReadFile(path) + if os.IsNotExist(err) { + snapshot[path] = nil + continue + } + if err != nil { + t.Fatalf("read sqlite file %s: %v", path, err) + } + snapshot[path] = append([]byte(nil), data...) + } + return snapshot +} + +func assertSQLiteFilesUnchanged(t *testing.T, dbPath string, before map[string][]byte) { + t.Helper() + after := snapshotSQLiteFiles(t, dbPath) + for path, want := range before { + if !bytes.Equal(after[path], want) { + t.Fatalf("sqlite file %s changed: before %d bytes after %d bytes", path, len(want), len(after[path])) + } + } +} + +func equalStrings(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} diff --git a/cmd/backscroll/index_policy.go b/cmd/backscroll/index_policy.go new file mode 100644 index 0000000..82c22b5 --- /dev/null +++ b/cmd/backscroll/index_policy.go @@ -0,0 +1,253 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/pablontiv/backscroll/internal/compat" + "github.com/pablontiv/backscroll/internal/config" + "github.com/pablontiv/backscroll/internal/storage" +) + +type indexCommandClass uint8 + +const ( + indexDataRead indexCommandClass = iota + indexMutation + indexDiagnostic + indexRemediation +) + +func prepareIndex(ctx context.Context, cfg *config.Config, class indexCommandClass, autoSync bool) (*storage.Database, *compat.Diagnostic, error) { + if cfg == nil { + return nil, &compat.Diagnostic{Code: compat.CodeIndexStale, Summary: "index configuration is unavailable"}, fmt.Errorf("index configuration is unavailable") + } + activePath, err := resolveActiveIndexPath(cfg.DatabasePath) + if err != nil { + d := continuationFor(compat.Diagnostic{Code: compat.CodeIndexStale, Summary: fmt.Sprintf("resolve active index path: %v", err)}, activePath) + return nil, &d, err + } + + if class == indexDataRead && !autoSync { + if _, statErr := os.Stat(cfg.DatabasePath); os.IsNotExist(statErr) { + return nil, nil, fmt.Errorf("backscroll database not found: %s: %w", cfg.DatabasePath, statErr) + } else if statErr != nil { + return nil, nil, fmt.Errorf("stat database: %w", statErr) + } + } + + openPrepared := func() (*storage.Database, *compat.Diagnostic, error) { + switch class { + case indexDataRead: + if !autoSync { + return openReadOnlyCurrentIndex(ctx, cfg.DatabasePath) + } + return storage.OpenCompatible(ctx, cfg.DatabasePath) + case indexMutation: + return storage.OpenCompatible(ctx, cfg.DatabasePath) + case indexDiagnostic, indexRemediation: + return openImmutableCurrentIndex(ctx, cfg.DatabasePath) + default: + return nil, nil, fmt.Errorf("unknown index command class %d", class) + } + } + + db, diag, err := openPrepared() + if diag != nil { + d := continuationFor(*diag, activePath) + return nil, &d, nil + } + if err != nil { + if errors.Is(err, storage.ErrImmutableReadOnlyWALUnsafe) { + d := compat.Diagnostic{ + Code: compat.CodeIndexStale, + Summary: fmt.Sprintf("current index snapshot cannot be inspected without side effects while its WAL has uncheckpointed frames; close the writer or checkpoint the database, then retry: %v", err), + } + return nil, &d, err + } + d := continuationFor(compat.Diagnostic{Code: compat.CodeMigrationFailed, Summary: fmt.Sprintf("prepare index failed: %v", err)}, activePath) + return nil, &d, err + } + + if autoSync { + if closeErr := db.Close(); closeErr != nil { + d := continuationFor(compat.Diagnostic{Code: compat.CodeIndexStale, Summary: fmt.Sprintf("close prepared index before sync: %v", closeErr)}, activePath) + return nil, &d, closeErr + } + if err := maybeAutoSync(cfg); err != nil { + d := continuationFor(compat.Diagnostic{Code: compat.CodeIndexStale, Summary: fmt.Sprintf("index sync failed: %v", err)}, activePath) + return nil, &d, err + } + db, diag, err = openPrepared() + if diag != nil { + d := continuationFor(*diag, activePath) + return nil, &d, nil + } + if err != nil { + if errors.Is(err, storage.ErrImmutableReadOnlyWALUnsafe) { + d := compat.Diagnostic{ + Code: compat.CodeIndexStale, + Summary: fmt.Sprintf("current index snapshot cannot be inspected without side effects while its WAL has uncheckpointed frames; close the writer or checkpoint the database, then retry: %v", err), + } + return nil, &d, err + } + d := continuationFor(compat.Diagnostic{Code: compat.CodeMigrationFailed, Summary: fmt.Sprintf("prepare index after sync failed: %v", err)}, activePath) + return nil, &d, err + } + } + + return db, nil, nil +} + +func openReadOnlyCurrentIndex(ctx context.Context, path string) (*storage.Database, *compat.Diagnostic, error) { + return openCurrentIndexWith(ctx, path, storage.OpenReadOnly) +} + +func openImmutableCurrentIndex(ctx context.Context, path string) (*storage.Database, *compat.Diagnostic, error) { + return openCurrentIndexWith(ctx, path, storage.OpenImmutableReadOnly) +} + +func openCurrentIndexWith(ctx context.Context, path string, open func(string) (*storage.Database, error)) (*storage.Database, *compat.Diagnostic, error) { + db, err := open(path) + if err != nil { + return nil, nil, err + } + plan, diag, inspectErr := compat.InspectIndex(ctx, db.DB()) + if inspectErr != nil || diag != nil { + closeErr := closeIndexDB(db, inspectErr) + if diag != nil && closeErr != nil { + diag.Summary = strings.TrimSpace(diag.Summary) + "; " + closeErr.Error() + } + return nil, diag, closeErr + } + if len(plan.Steps) > 0 { + diag := &compat.Diagnostic{ + Code: compat.CodeIndexStale, + Summary: fmt.Sprintf("index schema %s has %d pending migration step(s)", plan.From.Signature, len(plan.Steps)), + } + if closeErr := closeIndexDB(db, nil); closeErr != nil { + diag.Summary += "; " + closeErr.Error() + return nil, diag, closeErr + } + return nil, diag, nil + } + return db, nil, nil +} + +func closeIndexDB(db *storage.Database, err error) error { + if db == nil { + return err + } + if closeErr := db.Close(); closeErr != nil { + return errors.Join(err, fmt.Errorf("close index database: %w", closeErr)) + } + return err +} + +func continuationFor(d compat.Diagnostic, activePath string) compat.Diagnostic { + if strings.TrimSpace(activePath) == "" { + d.Continuation = nil + if !strings.Contains(d.Summary, "recovery continuation unavailable") { + d.Summary = strings.TrimSpace(d.Summary) + " (recovery continuation unavailable: active path is empty)" + } + return d + } + d.Continuation = []string{"recover", "--from", activePath, "--dry-run"} + return d +} + +func writeDiagnostic(stdout, stderr io.Writer, d compat.Diagnostic, jsonMode bool) error { + if jsonMode { + payload := struct { + Code string `json:"code"` + Summary string `json:"summary"` + Continuation []string `json:"continuation_argv,omitempty"` + }{ + Code: string(d.Code), + Summary: d.Summary, + Continuation: d.Continuation, + } + return json.NewEncoder(stdout).Encode(payload) + } + _, err := fmt.Fprintf(stderr, "diagnostic: %s: %s\n", d.Code, strings.TrimSpace(d.Summary)) + if err != nil { + return err + } + if len(d.Continuation) > 0 { + _, err = fmt.Fprintf(stderr, "continuation: %s\n", strings.Join(d.Continuation, " ")) + } + return err +} + +func writeRobotDiagnostic(stdout io.Writer, d compat.Diagnostic) error { + if _, err := fmt.Fprintf(stdout, "diagnostic_code=%s\n", d.Code); err != nil { + return err + } + if _, err := fmt.Fprintf(stdout, "diagnostic_summary=%s\n", strings.TrimSpace(d.Summary)); err != nil { + return err + } + if len(d.Continuation) > 0 { + if _, err := fmt.Fprintf(stdout, "diagnostic_continuation_argv=%s\n", strings.Join(d.Continuation, " ")); err != nil { + return err + } + } + return nil +} + +func refuseIndex(stdout, stderr io.Writer, d compat.Diagnostic, jsonMode, robotMode bool) error { + var err error + if robotMode { + err = writeRobotDiagnostic(stdout, d) + } else { + err = writeDiagnostic(stdout, stderr, d, jsonMode) + } + if err != nil { + return err + } + return indexDiagnosticError{diagnostic: d} +} + +type indexDiagnosticError struct { + diagnostic compat.Diagnostic +} + +func (e indexDiagnosticError) Error() string { + return fmt.Sprintf("%s: %s", e.diagnostic.Code, strings.TrimSpace(e.diagnostic.Summary)) +} + +func indexPolicyMachineArgs(args []string) bool { + for _, arg := range args { + if arg == "--json" || arg == "--robot" || strings.HasPrefix(arg, "--json=") || strings.HasPrefix(arg, "--robot=") { + return true + } + } + return false +} + +func resolveActiveIndexPath(path string) (string, error) { + if strings.TrimSpace(path) == "" { + return "", fmt.Errorf("database path is empty") + } + abs, err := filepath.Abs(path) + if err != nil { + return "", err + } + if _, err := os.Lstat(abs); err != nil { + if errors.Is(err, fs.ErrNotExist) { + return abs, nil + } + return "", err + } + resolved, err := filepath.EvalSymlinks(abs) + if err != nil { + return "", err + } + return resolved, nil +} diff --git a/cmd/backscroll/index_policy_test.go b/cmd/backscroll/index_policy_test.go new file mode 100644 index 0000000..06eae0c --- /dev/null +++ b/cmd/backscroll/index_policy_test.go @@ -0,0 +1,526 @@ +package main + +import ( + "bytes" + "context" + "database/sql" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/pablontiv/backscroll/internal/compat" + "github.com/pablontiv/backscroll/internal/config" + "github.com/pablontiv/backscroll/internal/storage" +) + +func TestStaleIndexBlocksIndexBackedCommands(t *testing.T) { + cases := []struct { + name string + argv []string + mutation bool + }{ + {"search", []string{"search", "sentinel"}, false}, + {"search-indexed-only", []string{"search", "sentinel", "--indexed-only"}, false}, + {"search-json", []string{"search", "sentinel", "--json"}, false}, + {"search-robot", []string{"search", "sentinel", "--robot"}, false}, + {"list", []string{"list"}, false}, + {"list-json", []string{"list", "--json"}, false}, + {"patterns", []string{"patterns", "--kind", "commands"}, false}, + {"rebuild", []string{"rebuild"}, true}, + {"purge", []string{"purge", "--before", "2030-01-01"}, true}, + {"annotate", []string{"annotate", "--uuid", "u", "--kind", "correction", "--label", "x"}, true}, + } + + ran := 0 + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ran++ + dbPath := newUnsupportedIndexedConsumerDB(t) + before := readDBBytes(t, dbPath) + + var stdout, stderr bytes.Buffer + err := run(&stdout, &stderr, tc.argv) + if err == nil { + t.Fatalf("%v succeeded; stdout=%q stderr=%q", tc.argv, stdout.String(), stderr.String()) + } + + combined := stdout.String() + stderr.String() + if strings.Contains(combined, "sentinel") { + t.Fatalf("%v emitted cached sentinel output after stale refusal; stdout=%q stderr=%q", tc.argv, stdout.String(), stderr.String()) + } + if !strings.Contains(combined, string(compat.CodeUnsupportedLineage)) { + t.Fatalf("%v did not emit typed diagnostic %q; stdout=%q stderr=%q err=%v", tc.argv, compat.CodeUnsupportedLineage, stdout.String(), stderr.String(), err) + } + wantContinuation := []string{"recover", "--from", dbPath, "--dry-run"} + for _, part := range wantContinuation { + if !strings.Contains(combined, part) { + t.Fatalf("%v diagnostic missing continuation part %q; stdout=%q stderr=%q", tc.argv, part, stdout.String(), stderr.String()) + } + } + if strings.Contains(combined, "") || strings.Contains(combined, "--from --dry-run") { + t.Fatalf("%v emitted placeholder/empty continuation; stdout=%q stderr=%q", tc.argv, stdout.String(), stderr.String()) + } + + if tc.mutation { + after := readDBBytes(t, dbPath) + if !bytes.Equal(after, before) { + t.Fatalf("%v mutated the unsupported database before policy refusal", tc.argv) + } + } + }) + } + if ran != len(cases) { + t.Fatalf("ran %d cases, want %d", ran, len(cases)) + } +} + +func TestIndexedOnlyDoesNotBypassStaleBlock(t *testing.T) { + dbPath := newUnsupportedIndexedConsumerDB(t) + var stdout, stderr bytes.Buffer + err := run(&stdout, &stderr, []string{"search", "sentinel", "--indexed-only", "--source-path", "/sentinel/%"}) + if err == nil { + t.Fatalf("indexed-only search succeeded; stdout=%q stderr=%q", stdout.String(), stderr.String()) + } + combined := stdout.String() + stderr.String() + if !strings.Contains(combined, string(compat.CodeUnsupportedLineage)) { + t.Fatalf("missing stale diagnostic for %s; stdout=%q stderr=%q", dbPath, stdout.String(), stderr.String()) + } + if strings.Contains(combined, "sentinel") { + t.Fatalf("indexed-only/filter path emitted cached sentinel output; stdout=%q stderr=%q", stdout.String(), stderr.String()) + } +} + +func TestMachineModesCarryDiagnosticCodeAndContinuation(t *testing.T) { + for _, tc := range []struct { + name string + argv []string + }{ + {"json", []string{"search", "sentinel", "--json"}}, + {"robot", []string{"search", "sentinel", "--robot"}}, + } { + t.Run(tc.name, func(t *testing.T) { + dbPath := newUnsupportedIndexedConsumerDB(t) + var stdout, stderr bytes.Buffer + err := run(&stdout, &stderr, tc.argv) + if err == nil { + t.Fatalf("machine mode succeeded; stdout=%q stderr=%q", stdout.String(), stderr.String()) + } + if stderr.Len() != 0 { + t.Fatalf("machine diagnostic contaminated stderr: %q", stderr.String()) + } + if strings.Contains(stdout.String(), "sentinel") { + t.Fatalf("machine diagnostic emitted cached sentinel rows: %q", stdout.String()) + } + switch tc.name { + case "json": + var got struct { + Code string `json:"code"` + Continuation []string `json:"continuation_argv"` + } + if err := json.Unmarshal(stdout.Bytes(), &got); err != nil { + t.Fatalf("invalid JSON diagnostic %q: %v", stdout.String(), err) + } + assertDiagnosticFields(t, got.Code, got.Continuation, dbPath) + case "robot": + out := stdout.String() + if !strings.Contains(out, "diagnostic_code="+string(compat.CodeUnsupportedLineage)) { + t.Fatalf("robot diagnostic missing code: %q", out) + } + if !strings.Contains(out, "diagnostic_continuation_argv=recover --from "+dbPath+" --dry-run") { + t.Fatalf("robot diagnostic missing exact continuation: %q", out) + } + } + }) + } +} + +func TestIndexedOnlyRejectsPendingMigrationWithoutMutating(t *testing.T) { + dbPath := newFixtureIndexDB(t, "v12.sql") + before := readDBBytes(t, dbPath) + setIndexPolicyEnv(t, dbPath, t.TempDir()) + + var stdout, stderr bytes.Buffer + err := run(&stdout, &stderr, []string{"search", "sentinel", "--indexed-only"}) + if err == nil { + t.Fatalf("indexed-only pending migration succeeded; stdout=%q stderr=%q", stdout.String(), stderr.String()) + } + combined := stdout.String() + stderr.String() + if !strings.Contains(combined, string(compat.CodeIndexStale)) || !strings.Contains(combined, "pending migration") { + t.Fatalf("missing pending-migration diagnostic; stdout=%q stderr=%q err=%v", stdout.String(), stderr.String(), err) + } + if !bytes.Equal(readDBBytes(t, dbPath), before) { + t.Fatal("indexed-only pending migration mutated the database") + } +} + +func TestAutoSyncFailuresBlockCachedConsumers(t *testing.T) { + cases := []struct { + name string + setup func(t *testing.T, root string) + wantError string + }{ + { + name: "discovery", + setup: func(t *testing.T, root string) { + writeInputManifest(t, root, "claude", root, []string{"*.jsonl"}, []string{"["}) + writeFile(t, filepath.Join(root, "session.jsonl"), `{"type":"message","message":{"role":"user","content":"fresh"}}`+"\n") + }, + wantError: "discover input", + }, + { + name: "hash", + setup: func(t *testing.T, root string) { + writeInputManifest(t, root, "opencode", root, []string{"*.db"}, nil) + writeFile(t, filepath.Join(root, "not-sqlite.db"), "not sqlite") + }, + wantError: "hash", + }, + { + name: "parse", + setup: func(t *testing.T, root string) { + writeInputManifest(t, root, "opencode", root, []string{"*.db"}, nil) + path := filepath.Join(root, "opencode.db") + raw, err := sql.Open("sqlite", path) + if err != nil { + t.Fatalf("open opencode fixture: %v", err) + } + defer func() { _ = raw.Close() }() + if _, err := raw.Exec(`CREATE TABLE message (id TEXT PRIMARY KEY, session_id TEXT, data TEXT, time_created INTEGER, time_updated INTEGER)`); err != nil { + t.Fatalf("create message table: %v", err) + } + }, + wantError: "parse", + }, + { + name: "sync", + setup: func(t *testing.T, root string) { + writeInputManifest(t, root, "claude", root, []string{"*.jsonl"}, nil) + writeFile(t, filepath.Join(root, "session.jsonl"), `{"type":"message","message":{"role":"user","content":"fresh"}}`+"\n") + orig := maybeAutoSyncSyncFiles + maybeAutoSyncSyncFiles = func(*storage.Database, []storage.IndexedFile) error { return fmt.Errorf("injected sync failure") } + t.Cleanup(func() { maybeAutoSyncSyncFiles = orig }) + }, + wantError: "sync files", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + dbPath := newSupportedIndexedConsumerDB(t) + root := filepath.Join(t.TempDir(), "inputs-root") + if err := os.MkdirAll(root, 0o755); err != nil { + t.Fatalf("mkdir input root: %v", err) + } + setIndexPolicyEnv(t, dbPath, t.TempDir()) + tc.setup(t, root) + + var stdout, stderr bytes.Buffer + err := run(&stdout, &stderr, []string{"search", "sentinel"}) + if err == nil { + t.Fatalf("auto-sync %s failure succeeded; stdout=%q stderr=%q", tc.name, stdout.String(), stderr.String()) + } + combined := stdout.String() + stderr.String() + if !strings.Contains(combined, string(compat.CodeIndexStale)) || !strings.Contains(combined, tc.wantError) { + t.Fatalf("missing %s diagnostic; stdout=%q stderr=%q err=%v", tc.wantError, stdout.String(), stderr.String(), err) + } + if strings.Contains(combined, "sentinel") { + t.Fatalf("auto-sync %s failure emitted cached sentinel: stdout=%q stderr=%q", tc.name, stdout.String(), stderr.String()) + } + }) + } +} + +func TestMigrationFailureBlocksCachedConsumer(t *testing.T) { + dbPath := newFixtureIndexDB(t, "v8.sql") + raw, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatalf("open fixture: %v", err) + } + _, err = raw.Exec(` + INSERT INTO tool_events (message_uuid, source_path, ordinal, tool_name, command_head, is_error, extraction_version) + VALUES ('dup-tool-uuid', '/dup-a.jsonl', 1, 'Bash', 'echo one', 0, 8), + ('dup-tool-uuid', '/dup-b.jsonl', 2, 'Bash', 'echo two', 0, 8) + `) + closeErr := raw.Close() + if err != nil { + t.Fatalf("seed duplicate tool events: %v", err) + } + if closeErr != nil { + t.Fatalf("close fixture: %v", closeErr) + } + setIndexPolicyEnv(t, dbPath, t.TempDir()) + + var stdout, stderr bytes.Buffer + err = run(&stdout, &stderr, []string{"search", "sentinel"}) + if err == nil { + t.Fatalf("migration failure succeeded; stdout=%q stderr=%q", stdout.String(), stderr.String()) + } + combined := stdout.String() + stderr.String() + if !strings.Contains(combined, string(compat.CodeMigrationFailed)) || !strings.Contains(combined, "conflicting tool_events") { + t.Fatalf("missing migration-failure diagnostic; stdout=%q stderr=%q err=%v", stdout.String(), stderr.String(), err) + } + if strings.Contains(combined, "sentinel") { + t.Fatalf("migration failure emitted cached sentinel: stdout=%q stderr=%q", stdout.String(), stderr.String()) + } +} + +func TestMachineModesSuppressAutoSyncProgressStderr(t *testing.T) { + dbPath := newSupportedIndexedConsumerDB(t) + root := filepath.Join(t.TempDir(), "inputs-root") + if err := os.MkdirAll(root, 0o755); err != nil { + t.Fatalf("mkdir input root: %v", err) + } + setIndexPolicyEnv(t, dbPath, t.TempDir()) + writeInputManifest(t, root, "claude", root, []string{"*.jsonl"}, nil) + writeFile(t, filepath.Join(root, "session.jsonl"), `{"type":"message","message":{"role":"user","content":"fresh machine"}}`+"\n") + + for _, argv := range [][]string{{"search", "fresh", "--json", "--all-projects"}, {"search", "fresh", "--robot", "--all-projects"}} { + t.Run(strings.Join(argv, " "), func(t *testing.T) { + var stdout, stderr bytes.Buffer + if err := run(&stdout, &stderr, argv); err != nil { + t.Fatalf("machine search failed: %v stdout=%q stderr=%q", err, stdout.String(), stderr.String()) + } + if stderr.Len() != 0 { + t.Fatalf("machine mode wrote progress to stderr: %q", stderr.String()) + } + }) + } +} + +func TestRebuildFailsOnDerivedMaintenanceError(t *testing.T) { + dbPath := newSupportedIndexedConsumerDB(t) + setIndexPolicyEnv(t, dbPath, t.TempDir()) + orig := rebuildBackfillDerived + rebuildBackfillDerived = func(*storage.Database, storage.BackfillDerivedOpts) error { + return fmt.Errorf("injected derived failure") + } + t.Cleanup(func() { rebuildBackfillDerived = orig }) + + var stdout, stderr bytes.Buffer + err := run(&stdout, &stderr, []string{"rebuild"}) + if err == nil { + t.Fatalf("rebuild succeeded despite derived failure; stdout=%q stderr=%q", stdout.String(), stderr.String()) + } + if strings.Contains(stderr.String(), "warning:") { + t.Fatalf("derived failure was downgraded to warning: %q", stderr.String()) + } + if !strings.Contains(err.Error(), "backfill derived") { + t.Fatalf("error = %v, want backfill derived", err) + } +} + +func TestResolveActiveIndexPathPropagatesBrokenSymlink(t *testing.T) { + dir := t.TempDir() + broken := filepath.Join(dir, "broken.db") + if err := os.Symlink(filepath.Join(dir, "missing-target.db"), broken); err != nil { + t.Skipf("symlink unavailable: %v", err) + } + _, diag, err := prepareIndex(context.Background(), &config.Config{DatabasePath: broken}, indexDataRead, true) + if err == nil { + t.Fatalf("broken symlink resolved successfully; diagnostic=%+v", diag) + } + if diag == nil || !strings.Contains(diag.Summary, "no such file") { + t.Fatalf("diagnostic=%+v err=%v, want EvalSymlinks error", diag, err) + } +} + +func assertDiagnosticFields(t *testing.T, code string, continuation []string, dbPath string) { + t.Helper() + if code != string(compat.CodeUnsupportedLineage) { + t.Fatalf("code=%q, want %q", code, compat.CodeUnsupportedLineage) + } + want := []string{"recover", "--from", dbPath, "--dry-run"} + if len(continuation) != len(want) { + t.Fatalf("continuation=%v, want %v", continuation, want) + } + for i := range want { + if continuation[i] != want[i] { + t.Fatalf("continuation=%v, want %v", continuation, want) + } + } +} + +func newSupportedIndexedConsumerDB(t *testing.T) string { + t.Helper() + dir := t.TempDir() + dbPath := filepath.Join(dir, "index.db") + db, err := storage.Open(dbPath) + if err != nil { + t.Fatalf("open seed db: %v", err) + } + messages := []storage.IndexedMessage{ + {Ordinal: 0, UUID: "u", Role: "user", Text: "sentinel cached text", Timestamp: "2026-01-01T00:00:00Z", ContentType: "text", ExtractionVersion: storage.CurrentExtractionVersion}, + {Ordinal: 1, UUID: "tool-u", Role: "assistant", Text: "sentinel cached command", Timestamp: "2026-01-01T00:00:01Z", ContentType: "tool", ToolName: "Bash", CommandHead: "sentinel", ExtractionVersion: storage.CurrentExtractionVersion}, + } + if err := db.SyncFiles([]storage.IndexedFile{{ + SourcePath: "/fixture/session.jsonl", + Source: "session", + Hash: "hash1", + Project: "project", + Messages: messages, + Tags: []string{"policy"}, + }}); err != nil { + _ = db.Close() + t.Fatalf("seed sync: %v", err) + } + if _, err := db.DB().Exec(`PRAGMA wal_checkpoint(TRUNCATE)`); err != nil { + _ = db.Close() + t.Fatalf("checkpoint: %v", err) + } + if err := db.Close(); err != nil { + t.Fatalf("close seed db: %v", err) + } + resolved, err := filepath.EvalSymlinks(dbPath) + if err != nil { + t.Fatalf("resolve db path: %v", err) + } + return resolved +} + +func newFixtureIndexDB(t *testing.T, fixture string) string { + t.Helper() + dbPath := filepath.Join(t.TempDir(), fixture+".db") + scriptPath := filepath.Join("..", "..", "internal", "compat", "testdata", "release-schemas", fixture) + script, err := os.ReadFile(scriptPath) + if err != nil { + t.Fatalf("read fixture %s: %v", fixture, err) + } + raw, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatalf("open fixture db: %v", err) + } + if _, err := raw.Exec(string(script)); err != nil { + _ = raw.Close() + t.Fatalf("execute fixture %s: %v", fixture, err) + } + if err := raw.Close(); err != nil { + t.Fatalf("close fixture db: %v", err) + } + resolved, err := filepath.EvalSymlinks(dbPath) + if err != nil { + t.Fatalf("resolve fixture db: %v", err) + } + return resolved +} + +func setIndexPolicyEnv(t *testing.T, dbPath, cfgDir string) { + t.Helper() + t.Setenv("HOME", t.TempDir()) + t.Setenv("BACKSCROLL_CONFIG_DIR", cfgDir) + t.Setenv("BACKSCROLL_DATABASE_PATH", dbPath) + emptyInputs := filepath.Join(t.TempDir(), "empty-inputs") + if err := os.MkdirAll(emptyInputs, 0o755); err != nil { + t.Fatalf("mkdir empty inputs: %v", err) + } + t.Setenv("BACKSCROLL_SESSION_DIRS", emptyInputs) +} + +func writeInputManifest(t *testing.T, root, format string, discoverRoot string, include, exclude []string) { + t.Helper() + cfgDir := os.Getenv("BACKSCROLL_CONFIG_DIR") + if cfgDir == "" { + t.Fatal("BACKSCROLL_CONFIG_DIR not set") + } + inputsDir := filepath.Join(cfgDir, "backscroll", "inputs") + if err := os.MkdirAll(inputsDir, 0o755); err != nil { + t.Fatalf("mkdir inputs dir: %v", err) + } + quoteList := func(values []string) string { + parts := make([]string, 0, len(values)) + for _, v := range values { + parts = append(parts, fmt.Sprintf("%q", v)) + } + return "[" + strings.Join(parts, ", ") + "]" + } + manifest := fmt.Sprintf(`version = 1 +[[inputs]] +id = "policy" +source = "session" +active = true +[inputs.discover] +roots = [%q] +include = %s +exclude = %s +follow_symlinks = false +[inputs.decode] +format = %q +`, discoverRoot, quoteList(include), quoteList(exclude), format) + writeFile(t, filepath.Join(inputsDir, "policy.inputs.toml"), manifest) +} + +func writeFile(t *testing.T, path, contents string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("mkdir %s: %v", filepath.Dir(path), err) + } + if err := os.WriteFile(path, []byte(contents), 0o644); err != nil { + t.Fatalf("write %s: %v", path, err) + } +} + +func newUnsupportedIndexedConsumerDB(t *testing.T) string { + t.Helper() + dir := t.TempDir() + home := filepath.Join(dir, "home") + cfgDir := filepath.Join(dir, "config") + emptyInputs := filepath.Join(dir, "empty-inputs") + for _, path := range []string{home, cfgDir, emptyInputs} { + if err := os.MkdirAll(path, 0o755); err != nil { + t.Fatalf("mkdir %s: %v", path, err) + } + } + dbPath := filepath.Join(dir, "index.db") + t.Setenv("HOME", home) + t.Setenv("BACKSCROLL_CONFIG_DIR", cfgDir) + t.Setenv("BACKSCROLL_DATABASE_PATH", dbPath) + t.Setenv("BACKSCROLL_SESSION_DIRS", emptyInputs) + t.Chdir(dir) + + db, err := storage.Open(dbPath) + if err != nil { + t.Fatalf("open seed db: %v", err) + } + messages := []storage.IndexedMessage{ + {Ordinal: 0, UUID: "u", Role: "user", Text: "policy fixture text", Timestamp: "2026-01-01T00:00:00Z", ContentType: "text", ExtractionVersion: storage.CurrentExtractionVersion}, + {Ordinal: 1, UUID: "tool-u", Role: "assistant", Text: "go test ./...", Timestamp: "2026-01-01T00:00:01Z", ContentType: "tool", ToolName: "Bash", CommandHead: "sentinelcmd", ExtractionVersion: storage.CurrentExtractionVersion}, + } + if err := db.SyncFiles([]storage.IndexedFile{{ + SourcePath: "/fixture/session.jsonl", + Source: "session", + Hash: "hash1", + Project: "project", + Messages: messages, + Tags: []string{"policy"}, + }}); err != nil { + _ = db.Close() + t.Fatalf("seed sync: %v", err) + } + if _, err := db.DB().Exec(`CREATE TABLE unexpected_shape_marker (id INTEGER PRIMARY KEY)`); err != nil { + _ = db.Close() + t.Fatalf("make unsupported: %v", err) + } + if _, err := db.DB().Exec(`PRAGMA wal_checkpoint(TRUNCATE)`); err != nil { + _ = db.Close() + t.Fatalf("checkpoint: %v", err) + } + if err := db.Close(); err != nil { + t.Fatalf("close seed db: %v", err) + } + resolved, err := filepath.EvalSymlinks(dbPath) + if err != nil { + t.Fatalf("resolve db path: %v", err) + } + return resolved +} + +func readDBBytes(t *testing.T, path string) []byte { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read db bytes: %v", err) + } + return data +} diff --git a/cmd/backscroll/list.go b/cmd/backscroll/list.go index 8734b60..212cd43 100644 --- a/cmd/backscroll/list.go +++ b/cmd/backscroll/list.go @@ -1,9 +1,11 @@ package main import ( + "context" "encoding/json" "fmt" "io" + "os" "github.com/spf13/cobra" @@ -25,8 +27,9 @@ func newListCmd(stdout, stderr io.Writer) *cobra.Command { ) cmd := &cobra.Command{ - Use: "list", - Short: "List all indexed sessions", + Use: "list", + Short: "List all indexed sessions", + SilenceUsage: true, Long: `List displays all indexed sessions, optionally filtered by project. Use --project to filter to a single project. @@ -61,35 +64,35 @@ Use --json to output as JSON.`, func runList(stdout, stderr io.Writer, project string, allProjects bool, recent int, jsonFormat, robotFormat, indexedOnly bool, - order string, limit, offset int) error { + order string, limit, offset int) (retErr error) { cfg, err := config.Load() if err != nil { return fmt.Errorf("load config: %w", err) } - // Auto-sync before query unless --indexed-only is set - if !indexedOnly { - if err := maybeAutoSync(cfg); err != nil { - _, _ = fmt.Fprintf(stderr, "warning: auto-sync failed: %v; using cached index\n", err) + if indexedOnly { + if _, statErr := os.Stat(cfg.DatabasePath); os.IsNotExist(statErr) { + if jsonFormat { + _, _ = fmt.Fprintf(stdout, "{\"count\":0,\"sessions\":[]}\n") + } else { + _, _ = fmt.Fprintf(stdout, "No sessions found\n") + } + return nil } } - // Derive effective project from cwd if not explicitly set - project = effectiveProject(project, allProjects) - - // Open read-only database - // If DB doesn't exist yet, return an empty list. - db, err := storage.OpenReadOnly(cfg.DatabasePath) + db, diag, err := prepareIndex(context.Background(), cfg, indexDataRead, !indexedOnly) + if diag != nil { + return refuseIndex(stdout, stderr, *diag, jsonFormat, robotFormat) + } if err != nil { - if jsonFormat { - _, _ = fmt.Fprintf(stdout, "{\"count\":0,\"sessions\":[]}\n") - } else { - _, _ = fmt.Fprintf(stdout, "No sessions found\n") - } - return nil + return fmt.Errorf("prepare index: %w", err) } - defer func() { _ = db.Close() }() + defer func() { retErr = closeIndexDB(db, retErr) }() + + // Derive effective project from cwd if not explicitly set + project = effectiveProject(project, allProjects) // If v2 grammar flags are provided (input, order, limit, offset), use ListItemsV2 // Otherwise fall back to legacy ListSessions for backward compat diff --git a/cmd/backscroll/main.go b/cmd/backscroll/main.go index 4eb1264..910f63b 100644 --- a/cmd/backscroll/main.go +++ b/cmd/backscroll/main.go @@ -1,6 +1,7 @@ package main import ( + "errors" "fmt" "io" "os" @@ -14,7 +15,10 @@ var version = "dev" func main() { if err := run(os.Stdout, os.Stderr, os.Args[1:]); err != nil { - _, _ = fmt.Fprintln(os.Stderr, err) + var indexErr indexDiagnosticError + if !errors.As(err, &indexErr) { + _, _ = fmt.Fprintln(os.Stderr, err) + } os.Exit(1) } } @@ -40,6 +44,10 @@ func run(stdout, stderr io.Writer, args []string) error { }() rootCmd := buildRootCmd(stdout, stderr) + if indexPolicyMachineArgs(args) { + rootCmd.SilenceErrors = true + rootCmd.SilenceUsage = true + } rootCmd.SetArgs(args) err := rootCmd.Execute() @@ -83,6 +91,7 @@ query merges both by rank position (RRF).`, newStatusCmd(stdout, stderr), newConfigCmd(stdout, stderr), newAnnotateCmd(stdout, stderr), + newRecoverCmd(stdout, stderr), ) return root diff --git a/cmd/backscroll/main_test.go b/cmd/backscroll/main_test.go index f25b1fb..58ab724 100644 --- a/cmd/backscroll/main_test.go +++ b/cmd/backscroll/main_test.go @@ -24,50 +24,53 @@ func boolPtr(b bool) *bool { return &b } func testEnv(t *testing.T) (dbPath string, cleanup func()) { t.Helper() dir := t.TempDir() - dbPath = filepath.Join(dir, "test.db") - origDB := os.Getenv("BACKSCROLL_DATABASE_PATH") - origCfg := os.Getenv("BACKSCROLL_CONFIG_DIR") - _ = os.Setenv("BACKSCROLL_DATABASE_PATH", dbPath) - _ = os.Setenv("BACKSCROLL_CONFIG_DIR", dir) - return dbPath, func() { - if origDB == "" { - _ = os.Unsetenv("BACKSCROLL_DATABASE_PATH") - } else { - _ = os.Setenv("BACKSCROLL_DATABASE_PATH", origDB) - } - if origCfg == "" { - _ = os.Unsetenv("BACKSCROLL_CONFIG_DIR") - } else { - _ = os.Setenv("BACKSCROLL_CONFIG_DIR", origCfg) + homeDir := filepath.Join(dir, "home") + configDir := filepath.Join(dir, "config") + for _, path := range []string{homeDir, configDir} { + if err := os.MkdirAll(path, 0o755); err != nil { + t.Fatalf("mkdir isolated test env path %s: %v", path, err) } } + dbPath = filepath.Join(dir, "test.db") + t.Setenv("HOME", homeDir) + t.Setenv("BACKSCROLL_CONFIG_DIR", configDir) + t.Setenv("BACKSCROLL_DATABASE_PATH", dbPath) + db, err := storage.Open(dbPath) + if err != nil { + t.Fatalf("create isolated test database: %v", err) + } + if _, err := db.DB().Exec(`PRAGMA wal_checkpoint(TRUNCATE)`); err != nil { + _ = db.Close() + t.Fatalf("checkpoint isolated test database: %v", err) + } + if err := db.Close(); err != nil { + t.Fatalf("close isolated test database: %v", err) + } + return dbPath, func() {} } // setupSessionDir sets BACKSCROLL_SESSION_DIRS to a path for auto-sync testing. // Returns the original value for restoration. func setupSessionDir(t *testing.T, path string) func() { t.Helper() - origDirs := os.Getenv("BACKSCROLL_SESSION_DIRS") - _ = os.Setenv("BACKSCROLL_SESSION_DIRS", path) - return func() { - if origDirs == "" { - _ = os.Unsetenv("BACKSCROLL_SESSION_DIRS") - } else { - _ = os.Setenv("BACKSCROLL_SESSION_DIRS", origDirs) - } - } + t.Setenv("BACKSCROLL_SESSION_DIRS", path) + return func() {} } -// syncForTest is a v1->v2 migration helper. In v2, sync is not a root command -// but auto-sync happens before queries. This helper just sets up session dirs -// and returns a fake "success" to maintain test patterns. +// syncForTest is a v1->v2 migration helper. In v2, sync is not a root command. +// This helper performs the explicit test sync setup so read-only diagnostic +// commands do not need to auto-sync. func syncForTest(t *testing.T, args ...string) (string, string, error) { t.Helper() // Extract --path value if present for i, arg := range args { if arg == "--path" && i+1 < len(args) { _ = setupSessionDir(t, args[i+1]) - return "", "", nil + cfg, err := config.Load() + if err != nil { + return "", "", err + } + return "", "", maybeAutoSync(cfg) } } return "", "", nil @@ -136,7 +139,7 @@ func TestSyncAndSearch(t *testing.T) { // v2: search auto-syncs before querying. No explicit sync needed. - // Status should show indexed content (auto-syncs first) + // Status is read-only; it should still render the existing index summary. out, _, err := runCmd("status") if err != nil { t.Fatalf("status error: %v", err) @@ -1181,7 +1184,7 @@ func TestStatusJSONIndexUsable(t *testing.T) { // After syncing a session, usable must flip to true piDir := filepath.Dir(filepath.Join(fixturesDir(), "claude-tool-events.jsonl")) _, _, _ = syncForTest(t, "sync", "--path", piDir) - // v2: status without --indexed-only triggers auto-sync + // Status is read-only; the explicit sync above populated the index. out, _, err = runCmd("status", "--json") if err != nil { t.Fatalf("status after sync error: %v", err) @@ -1561,7 +1564,7 @@ roots = ["/home/shared/myproject"] t.Setenv("BACKSCROLL_CONFIG_DIR", cfgDir) t.Setenv("HOME", home) - // Run status to trigger auto-sync + // Status is read-only; search below is responsible for auto-syncing content. out, stderr, err := runCmd("status") if err != nil { t.Fatalf("status failed: %v; stderr: %s", err, stderr) diff --git a/cmd/backscroll/patterns.go b/cmd/backscroll/patterns.go index 99e561a..6328874 100644 --- a/cmd/backscroll/patterns.go +++ b/cmd/backscroll/patterns.go @@ -1,6 +1,7 @@ package main import ( + "context" "encoding/json" "fmt" "io" @@ -36,8 +37,9 @@ func newPatternsCmd(stdout, stderr io.Writer) *cobra.Command { ) cmd := &cobra.Command{ - Use: "patterns", - Short: "Discover deterministic patterns in tool events, templates, sequences, and corrections", + Use: "patterns", + Short: "Discover deterministic patterns in tool events, templates, sequences, and corrections", + SilenceUsage: true, Long: `Patterns computes census aggregations over tool events (commands, failures), error message templates, frequent tool-call sequences (PrefixSpan mining), and message-level corrections to expose actionable pattern candidates @@ -87,7 +89,7 @@ Use --indexed-only to skip auto-sync (read existing index only).`, func runPatterns(stdout, stderr io.Writer, kind string, project string, allProjects bool, tag string, limit, offset int, jsonFormat, robotFormat, indexedOnly bool, minSupport int, minConfidence float64, pending bool, batch int, - minLength, maxLength int, after, before string, trend bool) error { + minLength, maxLength int, after, before string, trend bool) (retErr error) { // Early flag validation before DB open validKinds := map[string]bool{ @@ -118,24 +120,20 @@ func runPatterns(stdout, stderr io.Writer, return fmt.Errorf("load config: %w", err) } - // Auto-sync unless --indexed-only - if !indexedOnly { - if err := maybeAutoSync(cfg); err != nil { - _, _ = fmt.Fprintf(stderr, "warning: auto-sync failed: %v; using cached index\n", err) - } + db, diag, err := prepareIndex(context.Background(), cfg, indexDataRead, !indexedOnly) + if diag != nil { + return refuseIndex(stdout, stderr, *diag, jsonFormat, robotFormat) + } + if err != nil { + return fmt.Errorf("prepare index: %w", err) } + defer func() { retErr = closeIndexDB(db, retErr) }() // Derive effective project if project == "" && !allProjects { project = effectiveProject(project, allProjects) } - db, err := storage.OpenReadOnly(cfg.DatabasePath) - if err != nil { - return fmt.Errorf("open database: %w", err) - } - defer func() { _ = db.Close() }() - opts := storage.AggregateOptions{ Project: project, Tag: tag, diff --git a/cmd/backscroll/purge.go b/cmd/backscroll/purge.go index 688d109..a492da2 100644 --- a/cmd/backscroll/purge.go +++ b/cmd/backscroll/purge.go @@ -1,21 +1,22 @@ package main import ( + "context" "fmt" "io" "github.com/spf13/cobra" "github.com/pablontiv/backscroll/internal/config" - "github.com/pablontiv/backscroll/internal/storage" ) func newPurgeCmd(stdout, stderr io.Writer) *cobra.Command { var before string cmd := &cobra.Command{ - Use: "purge", - Short: "Remove old indexed records", + Use: "purge", + Short: "Remove old indexed records", + SilenceUsage: true, Long: `Purge removes all indexed records with timestamps before the specified date. The date should be in YYYY-MM-DD format (e.g., 2024-01-15). @@ -32,7 +33,7 @@ Example: backscroll purge --before 2024-01-01`, return cmd } -func runPurge(stdout, stderr io.Writer, before string) error { +func runPurge(stdout, stderr io.Writer, before string) (retErr error) { if before == "" { return fmt.Errorf("--before date is required") } @@ -42,12 +43,14 @@ func runPurge(stdout, stderr io.Writer, before string) error { return fmt.Errorf("load config: %w", err) } - // Open database - db, err := storage.Open(cfg.DatabasePath) + db, diag, err := prepareIndex(context.Background(), cfg, indexMutation, false) + if diag != nil { + return refuseIndex(stdout, stderr, *diag, false, false) + } if err != nil { - return fmt.Errorf("open database: %w", err) + return fmt.Errorf("prepare index: %w", err) } - defer func() { _ = db.Close() }() + defer func() { retErr = closeIndexDB(db, retErr) }() // Purge records deleted, err := db.Purge(before) diff --git a/cmd/backscroll/rebuild.go b/cmd/backscroll/rebuild.go index d48bb61..2328afc 100644 --- a/cmd/backscroll/rebuild.go +++ b/cmd/backscroll/rebuild.go @@ -14,8 +14,9 @@ import ( func newRebuildCmd(stdout, stderr io.Writer) *cobra.Command { cmd := &cobra.Command{ - Use: "rebuild", - Short: "Rebuild the FTS search indexes from the database", + Use: "rebuild", + Short: "Rebuild the FTS search indexes from the database", + SilenceUsage: true, Long: `Rebuild re-derives the FTS search indexes from the database itself and runs an incremental sync. It never deletes indexed content: sessions whose source files have expired from disk are preserved (the database is the @@ -28,22 +29,35 @@ perennial event store). Use 'purge' to delete data explicitly.`, return cmd } -func runRebuild(stdout, stderr io.Writer) error { +var ( + rebuildBackfillDerived = func(db *storage.Database, opts storage.BackfillDerivedOpts) error { + return db.BackfillDerived(opts) + } + rebuildReresolveProjects = func(db *storage.Database, ctx context.Context, resolver func(string) string) (int64, error) { + return db.ReresolveProjects(ctx, resolver) + } + rebuildReresolveProjectsWithRegistry = func(db *storage.Database, ctx context.Context, registry projects.ProjectRegistry) (int64, error) { + return db.ReresolveProjectsWithRegistry(ctx, registry) + } +) + +func runRebuild(stdout, stderr io.Writer) (retErr error) { cfg, err := config.Load() if err != nil { return fmt.Errorf("load config: %w", err) } - db, err := storage.Open(cfg.DatabasePath) + db, diag, err := prepareIndex(context.Background(), cfg, indexMutation, true) + if diag != nil { + return refuseIndex(stdout, stderr, *diag, false, false) + } if err != nil { - return fmt.Errorf("open database: %w", err) + return fmt.Errorf("prepare index: %w", err) } + defer func() { retErr = closeIndexDB(db, retErr) }() + _, _ = fmt.Fprintf(stdout, "Re-deriving FTS indexes from database...\n") - err = db.RebuildFTS() - if closeErr := db.Close(); closeErr != nil && err == nil { - err = closeErr - } - if err != nil { + if err := db.RebuildFTS(); err != nil { return fmt.Errorf("rebuild FTS: %w", err) } @@ -55,7 +69,7 @@ func runRebuild(stdout, stderr io.Writer) error { signalsFound int eventsExtracted int } - if err := db.BackfillDerived(storage.BackfillDerivedOpts{ + if err := rebuildBackfillDerived(db, storage.BackfillDerivedOpts{ OnProgress: func(processed, templateCount, signalCount, eventCount int) { backfillStats.filesProcessed = processed backfillStats.templatesFound = templateCount @@ -63,20 +77,13 @@ func runRebuild(stdout, stderr io.Writer) error { backfillStats.eventsExtracted = eventCount }, }); err != nil { - _, _ = fmt.Fprintf(stderr, "warning: backfill derived failed: %v\n", err) + return fmt.Errorf("backfill derived: %w", err) } if backfillStats.filesProcessed > 0 { _, _ = fmt.Fprintf(stdout, "Backfill complete: %d files processed, %d templates, %d corrections, %d lossy events.\n", backfillStats.filesProcessed, backfillStats.templatesFound, backfillStats.signalsFound, backfillStats.eventsExtracted) } - // Re-open for project resolution - db, err = storage.Open(cfg.DatabasePath) - if err != nil { - return fmt.Errorf("open database for re-resolution: %w", err) - } - defer func() { _ = db.Close() }() - // Re-resolve project identities from session paths _, _ = fmt.Fprintf(stdout, "Re-resolving projects from session paths...\n") resolver := func(sourcePath string) string { @@ -86,9 +93,9 @@ func runRebuild(stdout, stderr io.Writer) error { } return "" } - resolved, err := db.ReresolveProjects(context.Background(), resolver) + resolved, err := rebuildReresolveProjects(db, context.Background(), resolver) if err != nil { - _, _ = fmt.Fprintf(stderr, "warning: project re-resolution failed: %v\n", err) + return fmt.Errorf("project re-resolution: %w", err) } else if resolved > 0 { _, _ = fmt.Fprintf(stdout, "Re-resolved %d sessions with derived project identities.\n", resolved) } @@ -96,17 +103,13 @@ func runRebuild(stdout, stderr io.Writer) error { // Registry-aware re-resolution: correct historical fallback labels _, _ = fmt.Fprintf(stdout, "Checking registry for project label corrections...\n") registry := projects.LoadGlobalRegistry() - registryMatched, err := db.ReresolveProjectsWithRegistry(context.Background(), registry) + registryMatched, err := rebuildReresolveProjectsWithRegistry(db, context.Background(), registry) if err != nil { - _, _ = fmt.Fprintf(stderr, "warning: registry re-resolution failed: %v\n", err) + return fmt.Errorf("registry re-resolution: %w", err) } else if registryMatched > 0 { _, _ = fmt.Fprintf(stdout, "Registry matched and corrected %d sessions.\n", registryMatched) } - _, _ = fmt.Fprintf(stdout, "Running incremental sync...\n") - if err := maybeAutoSync(cfg); err != nil { - _, _ = fmt.Fprintf(stderr, "warning: sync failed: %v\n", err) - } _, _ = fmt.Fprintf(stdout, "Rebuild complete. No indexed data was deleted (perennity contract).\n") return nil } diff --git a/cmd/backscroll/recover.go b/cmd/backscroll/recover.go new file mode 100644 index 0000000..1899080 --- /dev/null +++ b/cmd/backscroll/recover.go @@ -0,0 +1,95 @@ +package main + +import ( + "context" + "fmt" + "io" + "strings" + + "github.com/pablontiv/backscroll/internal/config" + "github.com/pablontiv/backscroll/internal/recovery" + "github.com/spf13/cobra" +) + +func newRecoverCmd(stdout, stderr io.Writer) *cobra.Command { + var from string + var dryRun bool + fromValue := singleUseStringValue{target: &from} + + cmd := &cobra.Command{ + Use: "recover", + Short: "Recover stranded database rows into the configured database", + SilenceUsage: true, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + cfg, err := config.Load() + if err != nil { + return err + } + report, err := recovery.Execute(context.Background(), recovery.Options{ + ActivePath: cfg.DatabasePath, + FromPath: from, + DryRun: dryRun, + }) + if err != nil { + if backupPath, ok := recovery.RestorableBackupPath(err); ok { + _, _ = fmt.Fprintf(stderr, "manual recovery backup path: %s\n", backupPath) + } + return err + } + printRecoveryReport(stdout, report, dryRun) + return nil + }, + } + cmd.SetOut(stdout) + cmd.SetErr(stderr) + cmd.Flags().Var(&fromValue, "from", "path to one stranded Backscroll database") + cmd.Flags().BoolVar(&dryRun, "dry-run", false, "plan and report recovery without writing files") + _ = cmd.MarkFlagRequired("from") + return cmd +} + +type singleUseStringValue struct { + target *string + set bool +} + +func (v *singleUseStringValue) Set(value string) error { + if v.set { + return fmt.Errorf("flag --from may be set only once") + } + *v.target = value + v.set = true + return nil +} + +func (v *singleUseStringValue) String() string { + if v.target == nil { + return "" + } + return *v.target +} + +func (v *singleUseStringValue) Type() string { return "string" } + +func printRecoveryReport(w io.Writer, report recovery.Report, dryRun bool) { + if dryRun { + _, _ = fmt.Fprintln(w, "recovery dry run") + } else { + _, _ = fmt.Fprintln(w, "recovery complete") + } + _, _ = fmt.Fprintf(w, "active path: %s\n", report.ActivePath) + _, _ = fmt.Fprintf(w, "replacement target: %s\n", report.ActivePath) + _, _ = fmt.Fprintf(w, "backup path: %s\n", report.BackupPath) + for i, count := range report.InputCounts { + _, _ = fmt.Fprintf(w, "input %d records: %d\n", i+1, count) + } + _, _ = fmt.Fprintf(w, "exact duplicates: %d\n", report.ExactDuplicates) + _, _ = fmt.Fprintf(w, "final count: %d\n", report.FinalCount) + for i, shape := range report.Shapes { + _, _ = fmt.Fprintf(w, "input %d shape: version=%d signature=%s\n", i+1, shape.AppliedVersion, shape.Signature) + } + for _, conflict := range report.Conflicts { + _, _ = fmt.Fprintf(w, "diagnostic: %s: %s\n", conflict.Code, strings.TrimSpace(conflict.Summary)) + } +} diff --git a/cmd/backscroll/recover_test.go b/cmd/backscroll/recover_test.go new file mode 100644 index 0000000..95274bf --- /dev/null +++ b/cmd/backscroll/recover_test.go @@ -0,0 +1,491 @@ +package main + +import ( + "bytes" + "context" + "errors" + "fmt" + "os" + "path/filepath" + "reflect" + "sort" + "strings" + "testing" + "time" + + "github.com/pablontiv/backscroll/internal/compat" + "github.com/pablontiv/backscroll/internal/recovery" + "github.com/pablontiv/backscroll/internal/storage" +) + +type fileSnapshot struct { + Bytes []byte + Mode os.FileMode + MTime time.Time +} + +func TestRecoverDryRunMatchesUnionApplyPlanWithoutWrites(t *testing.T) { + dir := t.TempDir() + home := filepath.Join(dir, "home") + if err := os.MkdirAll(home, 0o755); err != nil { + t.Fatalf("create isolated HOME: %v", err) + } + activePath := filepath.Join(dir, "active.db") + fromPath := filepath.Join(dir, "stranded.db") + createRecoverTestDB(t, activePath, []storage.IndexedMessage{{ + Ordinal: 0, + Role: "user", + Text: "kept active row", + UUID: "11111111-1111-4111-8111-111111111111", + Timestamp: "2026-08-18T00:00:00Z", + ContentType: "text", + }}) + createRecoverTestDB(t, fromPath, []storage.IndexedMessage{ + { + Ordinal: 0, + Role: "user", + Text: "kept active row", + UUID: "11111111-1111-4111-8111-111111111111", + Timestamp: "2026-08-18T00:00:00Z", + ContentType: "text", + }, + { + Ordinal: 1, + Role: "assistant", + Text: "rescued stranded row", + UUID: "22222222-2222-4222-8222-222222222222", + Timestamp: "2026-08-18T00:01:00Z", + ContentType: "text", + }, + }) + + expected := expectedRecoveryPlan(t, activePath, fromPath) + if len(expected.InputShapes) != 2 { + t.Fatalf("expected recovery input shapes = %d, want 2", len(expected.InputShapes)) + } + before := inventoryDirectory(t, dir) + + t.Setenv("HOME", home) + t.Setenv("BACKSCROLL_CONFIG_DIR", filepath.Join(dir, "config")) + t.Setenv("BACKSCROLL_DATABASE_PATH", activePath) + t.Chdir(dir) + + var stdout, stderr bytes.Buffer + root := buildRootCmd(&stdout, &stderr) + root.SetArgs([]string{"recover", "--from", fromPath, "--dry-run"}) + err := root.Execute() + if err != nil { + t.Fatalf("recover dry run returned error: %v\nstderr=%s", err, stderr.String()) + } + if stderr.String() != "" { + t.Fatalf("stderr = %q, want empty", stderr.String()) + } + + after := inventoryDirectory(t, dir) + if !reflect.DeepEqual(after, before) { + t.Fatalf("dry run mutated active directory inventory\nbefore: %s\nafter: %s", describeInventory(before), describeInventory(after)) + } + + resolvedActivePath := canonicalRecoverTestPath(t, activePath) + out := stdout.String() + wantParts := []string{ + "recovery dry run", + "active path: " + resolvedActivePath, + "replacement target: " + resolvedActivePath, + "backup path: " + filepath.Join(filepath.Dir(resolvedActivePath), "."+filepath.Base(resolvedActivePath)+".backup--"), + "input 1 records: 1", + "input 2 records: 2", + "exact duplicates: 1", + "final count: 2", + fmt.Sprintf("input 1 shape: version=%d signature=%s", expected.InputShapes[0].AppliedVersion, expected.InputShapes[0].Signature), + fmt.Sprintf("input 2 shape: version=%d signature=%s", expected.InputShapes[1].AppliedVersion, expected.InputShapes[1].Signature), + } + for _, want := range wantParts { + if !strings.Contains(out, want) { + t.Fatalf("stdout missing %q\nstdout:\n%s", want, out) + } + } +} + +func TestRecoverCommandPreservesApplyFailureAs(t *testing.T) { + dir := t.TempDir() + home := filepath.Join(dir, "home") + if err := os.MkdirAll(home, 0o755); err != nil { + t.Fatalf("create isolated HOME: %v", err) + } + activePath := filepath.Join(dir, "active.db") + fromPath := filepath.Join(dir, "stranded.db") + sharedUUID := "99999999-9999-4999-8999-999999999999" + createRecoverTestDB(t, activePath, []storage.IndexedMessage{{Ordinal: 0, Role: "user", Text: "cli active conflict", UUID: sharedUUID, ContentType: "text"}}) + createRecoverTestDB(t, fromPath, []storage.IndexedMessage{{Ordinal: 0, Role: "user", Text: "cli stranded conflict", UUID: sharedUUID, ContentType: "text"}}) + + t.Setenv("HOME", home) + t.Setenv("BACKSCROLL_CONFIG_DIR", filepath.Join(dir, "config")) + t.Setenv("BACKSCROLL_DATABASE_PATH", activePath) + t.Chdir(dir) + + var stdout, stderr bytes.Buffer + root := buildRootCmd(&stdout, &stderr) + root.SetArgs([]string{"recover", "--from", fromPath}) + err := root.Execute() + if err == nil { + t.Fatal("recover command succeeded; want planning conflict error") + } + var failure *recovery.ApplyFailure + if !errors.As(err, &failure) { + t.Fatalf("command error %T %[1]v, want errors.As to *recovery.ApplyFailure", err) + } + if failure.Phase != recovery.ApplyFailurePhase("planning") { + t.Fatalf("ApplyFailure phase = %s, want planning", failure.Phase) + } +} + +func TestRecoverApplyReportsActualBackupAndCounts(t *testing.T) { + dir := t.TempDir() + home := filepath.Join(dir, "home") + if err := os.MkdirAll(home, 0o755); err != nil { + t.Fatalf("create isolated HOME: %v", err) + } + activePath := filepath.Join(dir, "active.db") + fromPath := filepath.Join(dir, "stranded.db") + createRecoverTestDB(t, activePath, []storage.IndexedMessage{{ + Ordinal: 0, + Role: "user", + Text: "cli active row", + UUID: "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", + Timestamp: "2026-08-18T00:00:00Z", + ContentType: "text", + }}) + createRecoverTestDB(t, fromPath, []storage.IndexedMessage{{ + Ordinal: 0, + Role: "assistant", + Text: "cli stranded row", + UUID: "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb", + Timestamp: "2026-08-18T00:01:00Z", + ContentType: "text", + }}) + activeBefore, err := os.ReadFile(activePath) + if err != nil { + t.Fatalf("read active before recovery: %v", err) + } + + t.Setenv("HOME", home) + t.Setenv("BACKSCROLL_CONFIG_DIR", filepath.Join(dir, "config")) + t.Setenv("BACKSCROLL_DATABASE_PATH", activePath) + t.Chdir(dir) + + var stdout, stderr bytes.Buffer + root := buildRootCmd(&stdout, &stderr) + root.SetArgs([]string{"recover", "--from", fromPath}) + if err := root.Execute(); err != nil { + t.Fatalf("recover apply returned error: %v\nstderr=%s", err, stderr.String()) + } + if stderr.String() != "" { + t.Fatalf("stderr = %q, want empty", stderr.String()) + } + + resolvedActivePath := canonicalRecoverTestPath(t, activePath) + out := stdout.String() + backupPath := recoverOutputValue(t, out, "backup path: ") + wantParts := []string{ + "recovery complete", + "active path: " + resolvedActivePath, + "replacement target: " + resolvedActivePath, + "backup path: " + backupPath, + "input 1 records: 1", + "input 2 records: 1", + "exact duplicates: 0", + "final count: 2", + } + for _, want := range wantParts { + if !strings.Contains(out, want) { + t.Fatalf("stdout missing %q\nstdout:\n%s", want, out) + } + } + backupBytes, err := os.ReadFile(backupPath) + if err != nil { + t.Fatalf("read reported backup %s: %v", backupPath, err) + } + if !bytes.Equal(backupBytes, activeBefore) { + t.Fatalf("reported backup bytes differ from original active bytes") + } +} + +func TestRecoverCommandReturnsStructuredApplyFailure(t *testing.T) { + dir := t.TempDir() + home := filepath.Join(dir, "home") + if err := os.MkdirAll(home, 0o755); err != nil { + t.Fatalf("create isolated HOME: %v", err) + } + missingActivePath := filepath.Join(dir, "missing-active.db") + fromPath := filepath.Join(dir, "stranded.db") + createRecoverTestDB(t, fromPath, []storage.IndexedMessage{{ + Ordinal: 0, + Role: "assistant", + Text: "cli stranded structured failure", + UUID: "cccccccc-cccc-4ccc-8ccc-cccccccccccc", + ContentType: "text", + }}) + + t.Setenv("HOME", home) + t.Setenv("BACKSCROLL_CONFIG_DIR", filepath.Join(dir, "config")) + t.Setenv("BACKSCROLL_DATABASE_PATH", missingActivePath) + t.Chdir(dir) + + var stdout, stderr bytes.Buffer + root := buildRootCmd(&stdout, &stderr) + root.SetArgs([]string{"recover", "--from", fromPath}) + err := root.Execute() + if err == nil { + t.Fatalf("recover apply succeeded; want structured failure") + } + var failure *recovery.ApplyFailure + if !errors.As(err, &failure) { + t.Fatalf("error %T %[1]v, want *recovery.ApplyFailure", err) + } + if failure.ActivePath == "" || failure.State == "" || failure.Phase == "" { + t.Fatalf("incomplete structured failure: %+v", failure) + } +} + +func TestRecoverCommandEmptyFromReturnsApplyFailure(t *testing.T) { + dir := t.TempDir() + home := filepath.Join(dir, "home") + if err := os.MkdirAll(home, 0o755); err != nil { + t.Fatalf("create isolated HOME: %v", err) + } + activePath := filepath.Join(dir, "active.db") + + t.Setenv("HOME", home) + t.Setenv("BACKSCROLL_CONFIG_DIR", filepath.Join(dir, "config")) + t.Setenv("BACKSCROLL_DATABASE_PATH", activePath) + t.Chdir(dir) + + var stdout, stderr bytes.Buffer + root := buildRootCmd(&stdout, &stderr) + root.SetArgs([]string{"recover", "--from", ""}) + err := root.Execute() + if err == nil { + t.Fatal("recover command succeeded; want structured missing --from failure") + } + var failure *recovery.ApplyFailure + if !errors.As(err, &failure) { + t.Fatalf("command error %T %[1]v, want *recovery.ApplyFailure", err) + } + if failure.Phase != recovery.ApplyFailurePhase("source-read") || failure.ActivePath == "" || failure.FromPath != "" { + t.Fatalf("ApplyFailure = %+v, want source-read with active and missing from", failure) + } +} + +func TestRecoverCommandPathCanonicalizationFailurePreservesApplyFailureAs(t *testing.T) { + dir := t.TempDir() + home := filepath.Join(dir, "home") + if err := os.MkdirAll(home, 0o755); err != nil { + t.Fatalf("create isolated HOME: %v", err) + } + activePath := filepath.Join(dir, "active.db") + fromPath := filepath.Join(dir, "broken-link.db") + createRecoverTestDB(t, activePath, []storage.IndexedMessage{{Ordinal: 0, Role: "user", Text: "cli active", UUID: "dddddddd-dddd-4ddd-8ddd-dddddddddddd", ContentType: "text"}}) + if err := os.Symlink(filepath.Join(dir, "missing-target.db"), fromPath); err != nil { + t.Skipf("symlink unsupported: %v", err) + } + + t.Setenv("HOME", home) + t.Setenv("BACKSCROLL_CONFIG_DIR", filepath.Join(dir, "config")) + t.Setenv("BACKSCROLL_DATABASE_PATH", activePath) + t.Chdir(dir) + + var stdout, stderr bytes.Buffer + root := buildRootCmd(&stdout, &stderr) + root.SetArgs([]string{"recover", "--from", fromPath}) + execErr := root.Execute() + if execErr == nil { + t.Fatal("recover command succeeded; want structured symlink failure") + } + var failure *recovery.ApplyFailure + if !errors.As(execErr, &failure) { + t.Fatalf("command error %T %[1]v, want *recovery.ApplyFailure", execErr) + } + wantFromPath, err := filepath.Abs(fromPath) + if err != nil { + t.Fatalf("absolute from path: %v", err) + } + if failure.FromPath != wantFromPath || failure.ActivePath != canonicalRecoverTestPath(t, activePath) { + t.Fatalf("ApplyFailure paths active=%q from=%q", failure.ActivePath, failure.FromPath) + } + if failure.Phase != recovery.ApplyFailurePhase("source-read") || !failure.NoActiveMutation { + t.Fatalf("ApplyFailure phase/no-active-mutation = %s/%v, want source-read without mutation", failure.Phase, failure.NoActiveMutation) + } +} + +func TestRecoverRejectsMissingFrom(t *testing.T) { + var stdout, stderr bytes.Buffer + cmd := buildRootCmd(&stdout, &stderr) + cmd.SetArgs([]string{"recover", "--dry-run"}) + err := cmd.Execute() + if err == nil { + t.Fatalf("recover without --from succeeded; stdout=%q stderr=%q", stdout.String(), stderr.String()) + } + if stdout.String() != "" { + t.Fatalf("stdout = %q, want empty", stdout.String()) + } + if !strings.Contains(err.Error(), `required flag(s) "from" not set`) && !strings.Contains(stderr.String(), `required flag(s) "from" not set`) { + t.Fatalf("missing --from error = %v, stderr=%q", err, stderr.String()) + } +} + +func TestRecoverHasNoGeneralMergeFlags(t *testing.T) { + tests := []struct { + name string + args []string + }{ + {name: "into", args: []string{"recover", "--from", "one.db", "--into", "two.db", "--dry-run"}}, + {name: "force", args: []string{"recover", "--from", "one.db", "--force", "--dry-run"}}, + {name: "partial", args: []string{"recover", "--from", "one.db", "--partial", "--dry-run"}}, + {name: "merge", args: []string{"recover", "--from", "one.db", "--merge", "--dry-run"}}, + {name: "skip", args: []string{"recover", "--from", "one.db", "--skip", "--dry-run"}}, + {name: "repeated-from", args: []string{"recover", "--from", "one.db", "--from", "two.db", "--dry-run"}}, + {name: "positional-arg", args: []string{"recover", "--from", "one.db", "--dry-run", "extra.db"}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var stdout, stderr bytes.Buffer + cmd := buildRootCmd(&stdout, &stderr) + cmd.SetArgs(tt.args) + err := cmd.Execute() + if err == nil { + t.Fatalf("%v succeeded; stdout=%q stderr=%q", tt.args, stdout.String(), stderr.String()) + } + if stdout.String() != "" { + t.Fatalf("stdout = %q, want empty", stdout.String()) + } + }) + } +} + +func createRecoverTestDB(t *testing.T, path string, messages []storage.IndexedMessage) { + t.Helper() + db, err := storage.Open(path) + if err != nil { + t.Fatalf("open test database %s: %v", path, err) + } + defer func() { + if err := db.Close(); err != nil { + t.Fatalf("close test database %s: %v", path, err) + } + }() + + if err := db.SyncFiles([]storage.IndexedFile{{ + SourcePath: "/sessions/shared.jsonl", + Source: "session", + Hash: "hash-" + filepath.Base(path), + Project: "project", + Messages: messages, + }}); err != nil { + t.Fatalf("sync test database %s: %v", path, err) + } +} + +func expectedRecoveryPlan(t *testing.T, paths ...string) compat.RecoveryPlan { + t.Helper() + inputs := make([]compat.RecoveryInput, 0, len(paths)) + for _, path := range paths { + db, err := storage.OpenImmutableReadOnly(path) + if err != nil { + t.Fatalf("open immutable readonly %s: %v", path, err) + } + input, diag, err := storage.ReadRecoveryInput(context.Background(), db) + closeErr := db.Close() + if err != nil { + t.Fatalf("read recovery input %s: %v", path, err) + } + if diag != nil { + t.Fatalf("read recovery input %s diagnostic: %+v", path, diag) + } + if closeErr != nil { + t.Fatalf("close readonly %s: %v", path, closeErr) + } + inputs = append(inputs, input) + } + plan, diagnostics, err := compat.PlanRecovery(inputs) + if err != nil { + t.Fatalf("PlanRecovery: %v", err) + } + if len(diagnostics) != 0 { + t.Fatalf("PlanRecovery diagnostics: %+v", diagnostics) + } + return plan +} + +func canonicalRecoverTestPath(t *testing.T, path string) string { + t.Helper() + abs, err := filepath.Abs(path) + if err != nil { + t.Fatalf("absolute path for %s: %v", path, err) + } + resolved, err := filepath.EvalSymlinks(abs) + if err != nil { + t.Fatalf("resolve path for %s: %v", abs, err) + } + return resolved +} + +func inventoryDirectory(t *testing.T, root string) map[string]fileSnapshot { + t.Helper() + inventory := map[string]fileSnapshot{} + if err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() { + return nil + } + info, err := entry.Info() + if err != nil { + return err + } + bytes, err := os.ReadFile(path) + if err != nil { + return err + } + rel, err := filepath.Rel(root, path) + if err != nil { + return err + } + inventory[rel] = fileSnapshot{Bytes: bytes, Mode: info.Mode(), MTime: info.ModTime()} + return nil + }); err != nil { + t.Fatalf("inventory %s: %v", root, err) + } + return inventory +} + +func recoverOutputValue(t *testing.T, output, prefix string) string { + t.Helper() + for _, line := range strings.Split(output, "\n") { + if strings.HasPrefix(line, prefix) { + value := strings.TrimSpace(strings.TrimPrefix(line, prefix)) + if value == "" { + t.Fatalf("output line %q has empty value", line) + } + return value + } + } + t.Fatalf("output missing prefix %q\noutput:\n%s", prefix, output) + return "" +} + +func describeInventory(inventory map[string]fileSnapshot) string { + paths := make([]string, 0, len(inventory)) + for path := range inventory { + paths = append(paths, path) + } + sort.Strings(paths) + parts := make([]string, 0, len(paths)) + for _, path := range paths { + entry := inventory[path] + parts = append(parts, fmt.Sprintf("%s bytes=%d mode=%s mtime=%s", path, len(entry.Bytes), entry.Mode, entry.MTime.Format(time.RFC3339Nano))) + } + return strings.Join(parts, "; ") +} diff --git a/cmd/backscroll/search.go b/cmd/backscroll/search.go index 5ce6a40..d9e4c1e 100644 --- a/cmd/backscroll/search.go +++ b/cmd/backscroll/search.go @@ -1,6 +1,7 @@ package main import ( + "context" "fmt" "io" "strings" @@ -12,7 +13,6 @@ import ( "github.com/pablontiv/backscroll/internal/config" "github.com/pablontiv/backscroll/internal/models" - "github.com/pablontiv/backscroll/internal/storage" ) func newSearchCmd(stdout, stderr io.Writer) *cobra.Command { @@ -39,8 +39,9 @@ func newSearchCmd(stdout, stderr io.Writer) *cobra.Command { ) cmd := &cobra.Command{ - Use: "search []", - Short: "Full-text search indexed content", + Use: "search []", + Short: "Full-text search indexed content", + SilenceUsage: true, Long: `Search performs a hybrid search (BM25 + vector embeddings with RRF fusion) across all indexed sessions, plans, and external sources. @@ -102,7 +103,7 @@ func runSearch(stdout, stderr io.Writer, source, sourcePath, after, before, role string, limit, offset int, contentType, tag string, fields string, maxTokens int, - lexicalOnly bool, similarityThreshold float64, indexedOnly bool) error { + lexicalOnly bool, similarityThreshold float64, indexedOnly bool) (retErr error) { // Validate flag values before opening the database if fields != "minimal" && fields != "full" { @@ -120,30 +121,25 @@ func runSearch(stdout, stderr io.Writer, return fmt.Errorf("invalid --content-type %q; must be one of: text, code, tool, reasoning", contentType) } - warnShortToolQuery(stderr, contentType, query) - cfg, err := config.Load() if err != nil { return fmt.Errorf("load config: %w", err) } - // Auto-sync before query unless --indexed-only is set - if !indexedOnly { - if err := maybeAutoSync(cfg); err != nil { - _, _ = fmt.Fprintf(stderr, "warning: auto-sync failed: %v; using cached index\n", err) - } + db, diag, err := prepareIndex(context.Background(), cfg, indexDataRead, !indexedOnly) + if diag != nil { + return refuseIndex(stdout, stderr, *diag, jsonFormat, robotFormat) + } + if err != nil { + return fmt.Errorf("prepare index: %w", err) } + defer func() { retErr = closeIndexDB(db, retErr) }() + + warnShortToolQuery(stderr, contentType, query) // Derive effective project from cwd if not explicitly set project = effectiveProject(project, allProjects) - // Open read-only database - db, err := storage.OpenReadOnly(cfg.DatabasePath) - if err != nil { - return fmt.Errorf("open database: %w", err) - } - defer func() { _ = db.Close() }() - // Parse dates var afterTime, beforeTime *time.Time if after != "" { diff --git a/cmd/backscroll/search_test.go b/cmd/backscroll/search_test.go index 5ef1ec1..bd6c551 100644 --- a/cmd/backscroll/search_test.go +++ b/cmd/backscroll/search_test.go @@ -236,6 +236,11 @@ func TestSearchWithJSONAndMaxTokens(t *testing.T) { } func TestSearchValidatesContentType(t *testing.T) { + _, cleanup := testEnv(t) + defer cleanup() + t.Setenv("HOME", t.TempDir()) + t.Setenv("BACKSCROLL_SESSION_DIRS", t.TempDir()) + tests := []struct { flag string wantErr bool diff --git a/cmd/backscroll/status.go b/cmd/backscroll/status.go index b771556..367202b 100644 --- a/cmd/backscroll/status.go +++ b/cmd/backscroll/status.go @@ -1,6 +1,7 @@ package main import ( + "context" "encoding/json" "fmt" "io" @@ -8,6 +9,7 @@ import ( "github.com/spf13/cobra" + "github.com/pablontiv/backscroll/internal/compat" "github.com/pablontiv/backscroll/internal/config" "github.com/pablontiv/backscroll/internal/input_config" "github.com/pablontiv/backscroll/internal/storage" @@ -29,41 +31,41 @@ func newStatusCmd(stdout, stderr io.Writer) *cobra.Command { - Configuration Use --json to output as JSON. -Use --indexed-only to skip auto-sync (read existing index only).`, +Status is read-only and never auto-syncs.`, RunE: func(cmd *cobra.Command, args []string) error { return runStatus(stdout, stderr, jsonFormat, indexedOnly) }, } cmd.Flags().BoolVar(&jsonFormat, "json", false, "Output as JSON") - cmd.Flags().BoolVar(&indexedOnly, "indexed-only", false, "Read existing index without auto-sync") + cmd.Flags().BoolVar(&indexedOnly, "indexed-only", false, "Deprecated: status is always read-only") return cmd } func runStatus(stdout, stderr io.Writer, jsonFormat, indexedOnly bool) error { + _ = indexedOnly cfg, err := config.Load() if err != nil { return fmt.Errorf("load config: %w", err) } - // Auto-sync before status unless --indexed-only is set - if !indexedOnly { - if err := maybeAutoSync(cfg); err != nil { - _, _ = fmt.Fprintf(stderr, "warning: auto-sync failed: %v; using cached index\n", err) - } - } - - // Check if database exists + // Check if database exists without creating it. Status is diagnostic/read-only + // and must not auto-sync or open the index through a writer. _, err = os.Stat(cfg.DatabasePath) dbExists := err == nil + if err != nil && !os.IsNotExist(err) { + return fmt.Errorf("stat database: %w", err) + } var stats storage.Stats if dbExists { - // Open database - db, err := storage.OpenReadOnly(cfg.DatabasePath) + db, diag, err := prepareIndex(context.Background(), cfg, indexDiagnostic, false) + if diag != nil { + return refuseDiagnostics(stdout, stderr, []compat.Diagnostic{*diag}, jsonFormat) + } if err != nil { - return fmt.Errorf("open database: %w", err) + return fmt.Errorf("open database read-only: %w", err) } defer func() { _ = db.Close() }() @@ -137,6 +139,8 @@ func runStatus(stdout, stderr io.Writer, jsonFormat, indexedOnly bool) error { } } else { _, _ = fmt.Fprintf(stdout, "\nIndex: Not yet created\n") + _, _ = fmt.Fprintf(stdout, " Files indexed: 0\n") + _, _ = fmt.Fprintf(stdout, " Messages indexed: 0\n") } _, _ = fmt.Fprintf(stdout, "\nConfiguration:\n") @@ -171,3 +175,40 @@ func resolveInputsForStatus(sessionDirs []string) ([]string, bool) { } return names, true } + +func recoveryDiagnosticsForIndex(db *storage.Database, activePath string) ([]compat.Diagnostic, error) { + input, diag, err := storage.ReadRecoveryInput(context.Background(), db) + if diag != nil { + d := continuationFor(*diag, activePath) + return []compat.Diagnostic{d}, err + } + if err != nil { + return nil, err + } + _, diagnostics, err := compat.PlanRecovery([]compat.RecoveryInput{input}) + if err != nil { + return nil, err + } + for i := range diagnostics { + diagnostics[i] = continuationFor(diagnostics[i], activePath) + } + return diagnostics, nil +} + +func refuseDiagnostics(stdout, stderr io.Writer, diagnostics []compat.Diagnostic, jsonMode bool) error { + if len(diagnostics) == 0 { + return nil + } + if jsonMode || len(diagnostics) == 1 { + if err := writeDiagnostic(stdout, stderr, diagnostics[0], jsonMode); err != nil { + return err + } + return indexDiagnosticError{diagnostic: diagnostics[0]} + } + for _, diagnostic := range diagnostics { + if err := writeDiagnostic(stdout, stderr, diagnostic, false); err != nil { + return err + } + } + return indexDiagnosticError{diagnostic: diagnostics[0]} +} diff --git a/cmd/backscroll/sync_helpers.go b/cmd/backscroll/sync_helpers.go index 86bae4c..edc0e72 100644 --- a/cmd/backscroll/sync_helpers.go +++ b/cmd/backscroll/sync_helpers.go @@ -2,7 +2,7 @@ package main import ( "fmt" - "os" + "io" "github.com/pablontiv/backscroll/internal/config" "github.com/pablontiv/backscroll/internal/input_config" @@ -13,17 +13,34 @@ import ( "github.com/pablontiv/backscroll/internal/templates" ) +var ( + maybeAutoSyncOpen = storage.Open + maybeAutoSyncActiveInputs = input_config.ActiveInputs + maybeAutoSyncLoadGlobalRegistry = projects.LoadGlobalRegistry + maybeAutoSyncNewRegistry = newDefaultAutoSyncRegistry + maybeAutoSyncSyncFiles = func(db *storage.Database, files []storage.IndexedFile) error { return db.SyncFiles(files) } + maybeAutoSyncProgress io.Writer = io.Discard +) + +func newDefaultAutoSyncRegistry() *readers.Registry { + reg := readers.NewRegistry() + reg.Register(&readers.OpenCodeReader{}) + reg.Register(&readers.ClaudeReader{}) + reg.Register(&readers.PiReader{}) + return reg +} + // maybeAutoSync performs an incremental sync operation if the database exists. // It is intended to be called before query commands to ensure fresh index state. // If sync fails, it returns an error (caller decides whether to warn/ignore). -func maybeAutoSync(cfg *config.Config) error { +func maybeAutoSync(cfg *config.Config) (retErr error) { // Open database for reading to check if it exists // (this will auto-create if missing) - db, err := storage.Open(cfg.DatabasePath) + db, err := maybeAutoSyncOpen(cfg.DatabasePath) if err != nil { return fmt.Errorf("open database: %w", err) } - defer func() { _ = db.Close() }() + defer func() { retErr = closeIndexDB(db, retErr) }() // Get existing file hashes existingHashes, err := db.GetFileHashes() @@ -34,9 +51,7 @@ func maybeAutoSync(cfg *config.Config) error { // Build stale-set once per run (files needing re-parse for rich metadata backfill) stalePaths, err := db.StalePaths(storage.CurrentExtractionVersion) if err != nil { - // Warn but continue if stale query fails - fmt.Fprintf(os.Stderr, "warning: stale paths query failed: %v\n", err) - stalePaths = nil + return fmt.Errorf("discover stale paths: %w", err) } staleSet := make(map[string]bool) for _, p := range stalePaths { @@ -47,19 +62,16 @@ func maybeAutoSync(cfg *config.Config) error { staleParsesDone := 0 // Build reader registry - reg := readers.NewRegistry() - reg.Register(&readers.OpenCodeReader{}) - reg.Register(&readers.ClaudeReader{}) - reg.Register(&readers.PiReader{}) + reg := maybeAutoSyncNewRegistry() // Resolve active inputs - defs, _, err := input_config.ActiveInputs(cfg.SessionDirs) + defs, _, err := maybeAutoSyncActiveInputs(cfg.SessionDirs) if err != nil { return fmt.Errorf("resolve inputs: %w", err) } // Load project registry - registry := projects.LoadGlobalRegistry() + registry := maybeAutoSyncLoadGlobalRegistry() // Collect indexed files var indexedFiles []storage.IndexedFile @@ -72,20 +84,18 @@ func maybeAutoSync(cfg *config.Config) error { reader, err := reg.ForDef(def) if err != nil { - // Warn but continue on reader errors - continue + return fmt.Errorf("resolve reader for input %q: %w", def.ID, err) } refs, err := reader.Discover(def) if err != nil { - // Warn but continue on discover errors - continue + return fmt.Errorf("discover input %q: %w", def.ID, err) } for _, ref := range refs { hash, err := reader.Hash(ref) if err != nil { - continue + return fmt.Errorf("hash %s: %w", ref, err) } // Skip unchanged files UNLESS they are in the stale-set and cap allows. @@ -95,12 +105,12 @@ func maybeAutoSync(cfg *config.Config) error { continue } staleParsesDone++ - fmt.Fprintf(os.Stderr, "Re-parsing stale file %d/%d: %s\n", staleParsesDone, len(stalePaths), ref) + _, _ = fmt.Fprintf(maybeAutoSyncProgress, "Re-parsing stale file %d/%d: %s\n", staleParsesDone, len(stalePaths), ref) } pf, err := reader.Parse(ref, def) if err != nil { - continue + return fmt.Errorf("parse %s: %w", ref, err) } // Use session cwd for project identification; fall back to file path if cwd is empty @@ -145,7 +155,7 @@ func maybeAutoSync(cfg *config.Config) error { // Sync all files if len(indexedFiles) > 0 { - if err := db.SyncFiles(indexedFiles); err != nil { + if err := maybeAutoSyncSyncFiles(db, indexedFiles); err != nil { return fmt.Errorf("sync files: %w", err) } } @@ -157,7 +167,7 @@ func maybeAutoSync(cfg *config.Config) error { const currentNormalizationVersion = 2 staleTemplatePaths, err := db.StaleTemplatePaths(currentNormalizationVersion) if err != nil { - fmt.Fprintf(os.Stderr, "warning: failed to discover stale templates: %v\n", err) + return fmt.Errorf("discover stale templates: %w", err) } else if len(staleTemplatePaths) > 0 { // Cap at staleTemplateCap per run; FIFO processing across runs if len(staleTemplatePaths) > staleTemplateCap { @@ -167,16 +177,15 @@ func maybeAutoSync(cfg *config.Config) error { for _, sourcePath := range staleTemplatePaths { msgs, err := db.LoadMessagesForPath(sourcePath) if err != nil { - fmt.Fprintf(os.Stderr, "warning: failed to load messages for %s: %v\n", sourcePath, err) - continue + return fmt.Errorf("load messages for stale template path %s: %w", sourcePath, err) } deletedCount, err := db.BackfillTemplatesForFile(miner, sourcePath, msgs) if err != nil { - fmt.Fprintf(os.Stderr, "warning: failed to re-mine templates for %s: %v\n", sourcePath, err) + return fmt.Errorf("re-mine templates for %s: %w", sourcePath, err) } if deletedCount > 0 { - fmt.Fprintf(os.Stderr, "Deleted %d stuck templates from %s\n", deletedCount, sourcePath) + _, _ = fmt.Fprintf(maybeAutoSyncProgress, "Deleted %d stuck templates from %s\n", deletedCount, sourcePath) } } } @@ -187,7 +196,7 @@ func maybeAutoSync(cfg *config.Config) error { // absent from indexed_files), so without this a detector fix never reaches it. // Bounded per run and convergent — see RederiveSupersededCorrections. if _, err := db.RederiveSupersededCorrections(staleTemplateCap); err != nil { - fmt.Fprintf(os.Stderr, "warning: failed to re-derive superseded correction signals: %v\n", err) + return fmt.Errorf("re-derive superseded correction signals: %w", err) } return nil diff --git a/cmd/backscroll/validate.go b/cmd/backscroll/validate.go index f4d5785..38c761e 100644 --- a/cmd/backscroll/validate.go +++ b/cmd/backscroll/validate.go @@ -1,17 +1,21 @@ package main import ( + "context" + "encoding/json" "fmt" "io" + "os" "github.com/spf13/cobra" + "github.com/pablontiv/backscroll/internal/compat" "github.com/pablontiv/backscroll/internal/config" - "github.com/pablontiv/backscroll/internal/storage" ) func newValidateCmd(stdout, stderr io.Writer) *cobra.Command { var indexedOnly bool + var jsonFormat bool cmd := &cobra.Command{ Use: "validate", @@ -23,43 +27,69 @@ func newValidateCmd(stdout, stderr io.Writer) *cobra.Command { Returns an error if validation fails. -Use --indexed-only to skip auto-sync (validate existing index only).`, +Validate is read-only and never auto-syncs.`, RunE: func(cmd *cobra.Command, args []string) error { - return runValidate(stdout, stderr, indexedOnly) + return runValidate(stdout, stderr, indexedOnly, jsonFormat) }, } - cmd.Flags().BoolVar(&indexedOnly, "indexed-only", false, "Read existing index without auto-sync") + cmd.Flags().BoolVar(&indexedOnly, "indexed-only", false, "Deprecated: validate is always read-only") + cmd.Flags().BoolVar(&jsonFormat, "json", false, "Output as JSON") return cmd } -func runValidate(stdout, stderr io.Writer, indexedOnly bool) error { +func runValidate(stdout, stderr io.Writer, indexedOnly bool, jsonFormat bool) error { + _ = indexedOnly cfg, err := config.Load() if err != nil { return fmt.Errorf("load config: %w", err) } - // Auto-sync before validate unless --indexed-only is set - if !indexedOnly { - if err := maybeAutoSync(cfg); err != nil { - _, _ = fmt.Fprintf(stderr, "warning: auto-sync failed: %v; validating cached index\n", err) + if _, err := os.Stat(cfg.DatabasePath); os.IsNotExist(err) { + if jsonFormat { + return json.NewEncoder(stdout).Encode(map[string]any{"valid": true, "database_exists": false}) } + _, _ = fmt.Fprintf(stdout, "✓ Index validation skipped: database not found\n") + return nil + } else if err != nil { + return fmt.Errorf("stat database: %w", err) } - // Open database - db, err := storage.Open(cfg.DatabasePath) + db, diag, err := prepareIndex(context.Background(), cfg, indexDiagnostic, false) + if diag != nil { + return refuseDiagnostics(stdout, stderr, []compat.Diagnostic{*diag}, jsonFormat) + } if err != nil { - return fmt.Errorf("open database: %w", err) + return fmt.Errorf("open database read-only: %w", err) } defer func() { _ = db.Close() }() - // Validate if err := db.Validate(); err != nil { + activePath, resolveErr := resolveActiveIndexPath(cfg.DatabasePath) + if resolveErr != nil { + return fmt.Errorf("resolve active index path: %w", resolveErr) + } + diagnostics, inspectErr := recoveryDiagnosticsForIndex(db, activePath) + if inspectErr != nil { + return fmt.Errorf("inspect recovery diagnostics: %w", inspectErr) + } + if len(diagnostics) > 0 { + return refuseDiagnostics(stdout, stderr, diagnostics, jsonFormat) + } + if jsonFormat { + if encodeErr := json.NewEncoder(stdout).Encode(map[string]any{"valid": false, "error": err.Error()}); encodeErr != nil { + return encodeErr + } + return err + } _, _ = fmt.Fprintf(stdout, "❌ Validation failed: %v\n", err) return err } + if jsonFormat { + return json.NewEncoder(stdout).Encode(map[string]any{"valid": true, "database_exists": true}) + } _, _ = fmt.Fprintf(stdout, "✓ Index validation passed\n") _, _ = fmt.Fprintf(stdout, "✓ All required tables exist\n") _, _ = fmt.Fprintf(stdout, "✓ FTS5 virtual table is set up correctly\n") diff --git a/docs/runbooks/long-running-agent-execution.md b/docs/runbooks/long-running-agent-execution.md new file mode 100644 index 0000000..93559e4 --- /dev/null +++ b/docs/runbooks/long-running-agent-execution.md @@ -0,0 +1,296 @@ +# Long-Running Agent Execution Runbook + +## 1) Purpose + +Use this runbook when a task is long-running, multi-step, or likely to need handoffs, reviews, retries, and durable state. + +Do not use it for tiny one-file fixes, trivial edits, or work that can be finished safely in one pass. + +## 2) Bounded task-review state machine + +- A task gets at most five fix-plus-scoped re-review rounds. +- Rounds 1–3 resume the original implementer. +- Rounds 4–5 dispatch a fresh, more capable implementer. +- After round 5 there is no ordinary round 6. Adjudicate every residual finding and record `Ruling: — — `; ask the human only if one of the four stop conditions applies. +- A wrong-worktree attempt that is stopped and quarantined before review does not count as a reviewed fix round. + +## 3) Preconditions and safety boundaries + +**Before starting:** + +- Confirm the exact worktree. +- Confirm the task brief, plan, optional spec, and current ledger. +- Start a durable ledger before editing. +- Exactly one implementation writer may be active globally, including work on disjoint files. +- Prefer read-only analysis first. +- Never rely on memory for current state. + +**Boundaries:** + +- Do not edit outside the assigned worktree. +- Do not stage, commit, push, merge, publish, or run destructive git commands unless the task explicitly allows it. +- Do not spawn subagents from workers unless the controller explicitly permits them. +- Do not let reviewers write code. +- Parallel activity is limited to read-only analysis that does not review an artifact currently being mutated. +- Freeze/stop mutation before independent review begins; mutation may resume only after review findings return. +- Review of an artifact must not overlap mutation of that artifact. + +## 4) Validated loop + +| Stage | What to do | Invariant | +|---|---|---| +| Recover context | Re-read the plan, optional spec, ledger, and latest progress/report notes | Fresh session can rebuild state from files alone | +| Isolate worktree | Verify resolved cwd, branch, worktree identity, and git status before mutation | No cross-worktree edits | +| Establish durable ledger | Create/update the ledger with plan identity, task state, owners, checkpoints, risks, and stop conditions | Ledger outranks memory | +| One global implementation writer | Keep exactly one implementation writer active globally; no second implementation writer, even on disjoint files | No write contention | +| Task brief | Give each writer one crisp objective and one file scope | No drift | +| Implement | Make the smallest change that advances the brief | Change stays local | +| Initial evidence | Before review, report exact focused test commands/results for the initial implementation | Claims are backed by output | +| Freeze package | Freeze tracked changes against BASE plus all new/untracked content, focused-test output, and ledger entry | Review reads a stable package, not the mutable worktree | +| Independent review | Use a separate read-only reviewer that checks spec and quality separately | Review is independent | +| Bounded fix loop | Apply only review-approved corrections | Fixes are narrow | +| Adjudicate residuals | Record `Ruling: — — ` for every residual finding; ask the human only if one of the exact four stop conditions triggers | Every residual is closed or escalated | +| Commit checkpoint | Commit a coherent slice only when COMMIT_POLICY permits and after approval or cap adjudication; otherwise record frozen checkpoint/hash/diff identity without committing | A known-good checkpoint exists | +| Continue without polling | Move to the next queued item; do not wait for human narration | Execution stays flowing | + +Per-task lifecycle: + +1. Record BASE. +2. Assign one fresh implementer and record active/original implementer identity. +3. Make the initial implementation. +4. Run focused tests and report exact commands/results before review. +5. Freeze the candidate package. +6. Run independent read-only review against the frozen package with separate spec and quality verdicts. +7. Fix only the scoped findings and re-review only that scope, up to the five-round cap. +8. Adjudicate every residual finding in the ledger with `Ruling: — — `. +9. Commit only when COMMIT_POLICY permits and only after approval or cap adjudication; when commits are forbidden, record the frozen checkpoint/hash/diff identity without committing. +10. Mark the ledger complete. +11. Proceed to the next task. + +Review packaging for uncommitted diffs: package the candidate as exact tracked changes against BASE plus all new/untracked content, focused-test output, and the current ledger entry. Reviewers inspect the frozen candidate package, not the mutable working copy, and never overlap mutation of the reviewed artifact. When COMMIT_POLICY forbids staging/committing, build the package without staging (for example, by saving `git diff`, `git ls-files --others --exclude-standard`, copied untracked file contents, command output, and ledger excerpts under an allowed artifact path). After all task loops, package the whole branch/range for independent final review. + +Mandatory final gates: run every final command mandated by PLAN plus repository gates (format/check/tests/coverage as applicable, using commands discovered from the repository rather than invented), record exact commands and outcomes, and if final review finds issues, use one bounded final fix wave and one scoped re-review. After that final fix wave, rerun every PLAN/repository final gate on the resulting final state and record fresh exact commands/results. Do not claim completion until the post-fix gates and final review have current evidence. + +This runbook intentionally uses review-before-commit because that is the observed lifecycle; keep that ordering explicit. + +## 5) Safe parallelism + +Safe to overlap: + +- Read-only analysis that does not review an artifact currently being mutated. +- Test exploration that does not write shared state. +- Parallel checks that remain read-only and stop before any independent review of a frozen artifact. + +Not safe to overlap: + +- Any second implementation writer, even on disjoint files. +- Any task that depends on another task’s uncommitted output. +- Review of an artifact while it is still being mutated. +- Independent review before mutation has been frozen and stopped. + +Rule of thumb: serialize all mutation; parallelize only read-only work that cannot conflict with the single active writer. + +## 6) Instruction patterns that worked + +### Bootstrap prompt + +```text +You are the fresh session controller for this task. + +Operational placeholders, to be filled before execution: +- WORKTREE= +- PLAN= +- SPEC= +- LEDGER= +- COMMIT_POLICY= +- QUARANTINE_PATH= + +1. Enter and verify the assigned checkout before any mutation: + - Run `cd "$WORKTREE" && pwd -P` and verify the resolved cwd equals the intended WORKTREE. + - Verify expected branch/worktree identity with `git branch --show-current`, `git rev-parse --show-toplevel`, and `git worktree list` (or the repository's equivalent identity check). + - Run `git status --short --branch` and record the result. + - If cwd, branch, worktree identity, or status is not the expected assigned state, do not mutate. Use the wrong-worktree incident procedure below. + +2. Read durable context before editing: + - Read PLAN. + - If SPEC is not "none", read SPEC. + - Read LEDGER if it exists. If LEDGER is absent, create it before mutation with the PLAN identity, SPEC identity, WORKTREE, COMMIT_POLICY, BASE placeholder, task list placeholder, and current timestamp/session identity. + - Ledger outranks memory. + +3. Maintain these global rules: + - Exactly one implementation writer may be active globally, including work on disjoint files. + - Parallel work is read-only only, and must not review an artifact currently being mutated. + - Freeze/stop mutation before independent review begins; mutation may resume only after review findings return. + - Reviewers are read-only and review the frozen package, not the mutable worktree. + - Do not spawn subagents unless explicitly permitted. + - Never push, merge, publish, edit outside WORKTREE, or cause another outside-worktree side effect without explicit permission; this is one of the stop conditions. + +4. Use the exact five-round task state machine: + - Each task gets at most five fix-plus-scoped re-review rounds. + - Rounds 1–3 resume the original implementer. + - Rounds 4–5 dispatch a fresh, more capable implementer. + - After round 5 there is no ordinary round 6. Adjudicate every residual finding and record `Ruling: — — `; ask the human only if one of the four stop conditions applies. + - A wrong-worktree attempt that is stopped and quarantined before review does not count as a reviewed fix round. + +5. For every task, persist these fields in LEDGER: + - BASE (`git rev-parse HEAD` or the approved base identifier before task mutation). + - Active implementer identity and original implementer identity. + - Current fix round number. + - Review spec verdict and review quality verdict. + - Every finding and its disposition. + - Focused test commands/results, including exact initial implementation focused test commands/results before review. + - Frozen candidate package path or identity. + - Commit checkpoint, or when COMMIT_POLICY forbids commits, the frozen checkpoint/hash/diff identity without a commit. + - Rulings in the exact form `Ruling: — — `. + +6. Implement and collect evidence: + - Record BASE before mutation. + - Assign exactly one implementer for one bounded task. + - Make the smallest local change that satisfies the brief. + - Before any review, run and report exact focused test commands/results for the initial implementation, even if they fail or are intentionally not run with a stated reason. + +7. Freeze the candidate for review: + - Stop mutation. + - Package exact tracked changes against BASE plus all new/untracked content, focused-test output, and the current LEDGER entry. + - The independent reviewer reads this frozen candidate package, not the mutable worktree. + - If COMMIT_POLICY forbids staging/committing, package safely without staging: save `git diff "$BASE"`, `git ls-files --others --exclude-standard`, copies or archived contents of each untracked file, focused-test output, and the LEDGER excerpt under an allowed artifact path. + +8. Review and fix: + - Run independent read-only review with separate spec and quality verdicts. + - Apply only scoped reviewer findings. + - After each fix, collect exact commands/results and focused tests, update LEDGER, freeze a new package, and run only the scoped re-review needed for the changed scope. + - At the cap, adjudicate every residual finding with a Ruling. + +9. Enforce COMMIT_POLICY: + - Commit only when COMMIT_POLICY permits it, and only after review approval or cap adjudication. + - When COMMIT_POLICY forbids commits, do not stage or commit; record the frozen checkpoint/hash/diff identity in LEDGER instead. + - Never push, merge, or publish without explicit permission; treat that as an outside-worktree stop. + +10. Wrong-worktree incident procedure: + - Stop the writer immediately. + - Inventory both the intended WORKTREE and the accidental checkout: resolved cwd, branch, `git status --short --branch`, and relevant changed/untracked paths. + - Never broad reset/clean; preserve pre-existing user changes. + - Quarantine only positively identified agent-created output under QUARANTINE_PATH. + - Do not touch files whose ownership is uncertain. If ownership is uncertain but cleanup/proceeding does not require an irreversible/destructive operation, record a Ruling and continue without touching uncertain files. Stop only when cleanup/proceeding would require an irreversible/destructive operation. + - Verify the intended candidate in WORKTREE before resuming. + - Record the incident, inventory, quarantine path, uncertainty disposition, and verification in LEDGER. + +11. Final sequence: + - After all task loops, package the whole branch/range for independent final review. + - Run every final command mandated by PLAN plus repository gates discovered from the repository (format/check/tests/coverage as applicable), and record exact commands/results. + - If final review finds issues, use one bounded final fix wave and one scoped re-review, then adjudicate residuals. + - After that final fix wave, rerun every PLAN/repository final gate on the resulting final state and record fresh exact commands/results. + - Do not claim success until final review and the post-fix gates are fresh and recorded in LEDGER. + +12. Stop only for these four conditions: + - irreversible/destructive operation; + - security-sensitive action; + - outside-worktree side effect normally requiring permission (merge/push/publish); + - plan so broken every path is a guess. + Ordinary ambiguity does not stop the run. Record `Ruling: — — ` and continue. +``` + +### Implementer prompt + +```text +You are the implementer for one bounded task. +Work only in the assigned worktree. +Do not spawn subagents unless the controller assigns them. +Do not edit outside the assigned file scope. +Read the task brief, ledger, and relevant plan notes first. +Make the smallest change that satisfies the brief. +Report exact commands/results, focused tests, and any residual risk. +``` + +### Reviewer prompt + +```text +You are the independent reviewer. +Read only; do not edit files. +Check the frozen candidate package against the task brief, ledger, plan, and optional spec. +Return separate spec and quality verdicts, plus concrete evidence. +``` + +### Fix-round prompt + +```text +Apply only the scoped reviewer findings. +Keep the patch minimal and local. +Do not rework unrelated code. +Re-run the narrow tests that prove the fix. +State clearly what changed and what was revalidated. +``` + +### Session-resume prompt + +```text +Resume from the ledger, not from memory. +First read: plan, progress notes, task reports, and the latest git status. +Then restate the current task queue, the active writer, and the next checkpoint. +If anything is ambiguous, record: `Ruling: — — ` and continue, unless a stop condition applies. +``` + +## 7) Anti-patterns observed + +| Anti-pattern | Why it caused churn | +|---|---| +| Wrong cwd / wrong worktree | Edits landed in the wrong checkout and had to be quarantined or restored. | +| Reviewer spawned by worker | Broke independence and blurred accountability. | +| Controller fixes code directly | Collapsed the separation between orchestration and implementation. | +| Unbounded review | Produced repeated nitpicks without a clear stop point. | +| Silent warnings | Deferred problems surfaced later as churn. | +| Trusting memory over ledger | Session state drifted and caused duplicate or conflicting work. | +| Concurrent writers | Created merge conflicts and ambiguous ownership. | + +## 8) Failure recovery playbook + +1. **Stop the writer.** Pause mutation immediately. +2. **Inventory both checkouts.** Identify the intended worktree and the stray checkout, including resolved cwd, branch, status, and changed/untracked paths. +3. **Do not broad reset/clean.** Preserve pre-existing changes. +4. **Quarantine only positively identified agent output** under the authorized quarantine path. +5. **Do not touch uncertain files.** If ownership is uncertain but cleanup/proceeding does not require an irreversible/destructive operation, record `Ruling: — — ` and continue without touching uncertain files. Stop only when cleanup/proceeding would require an irreversible/destructive operation. +6. **Verify the candidate worktree.** Re-read cwd, branch, plan, spec if any, ledger, and git status. +7. **Record the incident in the ledger.** Include inventories for intended and accidental checkouts, quarantine path, uncertainty disposition, and candidate verification. +8. **Reverify cwd/branch before resuming.** + +### Stop conditions + +Stop and escalate only for: + +- irreversible/destructive operation; +- security-sensitive action; +- outside-worktree side effect normally requiring permission (merge/push/publish); +- plan so broken every path is a guess. + +Ordinary ambiguity does not stop the run. Instead, record: `Ruling: — — ` and continue. + +## 9) Fresh-session launch checklist + +- [ ] Fill the canonical bootstrap placeholders: WORKTREE, PLAN, optional SPEC, LEDGER, COMMIT_POLICY, and QUARANTINE_PATH. +- [ ] Use the single canonical `### Bootstrap prompt` in section 6; do not copy or maintain a second bootstrap block. +- [ ] Confirm the assigned worktree by entering WORKTREE and verifying resolved cwd, branch/worktree identity, and git status before mutation. +- [ ] Read PLAN, SPEC if present, latest progress notes, relevant task reports, and LEDGER. +- [ ] Create LEDGER if absent. +- [ ] Verify exactly one or zero implementation writer globally. +- [ ] Define the next bounded task. +- [ ] Choose the smallest test that proves it. + +## 10) Evidence and limitations + +Final review packaging and gates: after all task loops, package the whole branch/range for independent final review; run every final command mandated by PLAN plus repository gates (format/check/tests/coverage as applicable, using commands discovered from the repository rather than invented); record exact commands and outcomes. If final review finds issues, use one bounded final fix wave and one scoped re-review, then adjudicate residuals; after that final fix wave, rerun every PLAN/repository final gate on the resulting final state and record fresh exact commands/results. Do not claim completion until final review and the post-fix gates have fresh evidence. + +This runbook is evidence-based, but it is derived from **one extended execution**. Treat the patterns here as observed guidance from that session, not universal law. + +**Observed facts from the session:** + +- Isolated worktrees reduced accidental cross-editing. +- A ledger made session recovery practical. +- Separate review improved correctness. +- Narrow fix loops prevented drift. + +**General rules inferred from the session:** + +- Serialize shared writes. +- Keep prompts short and explicit. +- Prefer file-backed state over memory. +- Validate before declaring success. diff --git a/docs/superpowers/plans/2026-08-18-index-lineage-compatibility.md b/docs/superpowers/plans/2026-08-18-index-lineage-compatibility.md new file mode 100644 index 0000000..5d58c5b --- /dev/null +++ b/docs/superpowers/plans/2026-08-18-index-lineage-compatibility.md @@ -0,0 +1,307 @@ +# Index Lineage Compatibility Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make every published Go index lineage through v3.2.5 inspectable and losslessly upgradeable with shape-safe, transactional migration primitives. + +**Architecture:** Add a narrow, stateless, read-only `internal/compat` inspector that classifies observed SQLite shape and returns typed diagnostics or ordered migration identifiers. Keep snapshots, transactions, migration SQL, and final verification in storage consumers. This slice deliberately does not activate stale-index refusal or expose a `recover` continuation; Plan 3 does both only after `recover` is registered and functional. + +**Tech Stack:** Go 1.26.2, `database/sql`, `modernc.org/sqlite`, table-driven Go tests, checked-in JSON/SQL fixtures, Just + +**Spec:** `docs/superpowers/specs/2026-08-18-systemic-index-compatibility-design.md` + +## Global Constraints + +- This is delivery Plan 1. The exact chain is Plan 1 inspection/migration primitives → Plan 3 recovery and blocking-policy activation → Plan 2 manifest ingestion → Plan 4 shipped guidance validation. +- `internal/compat` is stateless and read-only: no writes, transactions, backups, destination paths, Cobra output, retries, cached status, or workflow state. +- Plan 1 defines only schema-inspection and migration types required by this slice: `Code`, `Diagnostic`, `SchemaShape`, `MigrationStep`, `MigrationPlan`, and `Queryer`. +- Plan 1 does not define manifest plans, reader lookup, recovery inputs/plans, canonical recovery records, or move `storage.IndexedRecord`; those types belong to the first plan that consumes them. +- Actual tables, columns, indexes, triggers, and migration rows select a lineage; a migration version alone never does. +- Hermetic tests use only a checked-in release/schema manifest and fixtures covering published Go releases v0.3.7 through v3.2.5; they never consult ambient Git tags or the network. +- Frozen Rust v0 remains outside in-place migration. +- No user-facing command may refuse service or print `recover` in this slice. Unsupported shapes remain inspectable typed results for Plan 3 to render after the continuation exists. +- Every filesystem test uses `t.TempDir()` and no real HOME, config, Git tag state, or network. +- Follow strict RED → GREEN → TRIANGULATE → REFACTOR for every task. Run focused tests before `just check`, `just test`, and `just ci`. +- Commit commands below are instructions for a future explicitly authorized implementation. Do not stage or commit while authoring or reviewing this plan. +- Do not implement the reported large-file symptom without a focused fixture that fails on current code. + +--- + +## File map + +| Path | Responsibility | +|---|---| +| `internal/compat/types.go` | Core schema compatibility codes, diagnostics, shapes, migration plans, and the read-only query interface. | +| `internal/compat/catalog.go` | Embed and validate the checked-in release/schema inventory. | +| `internal/compat/catalog_test.go` | Hermetic inventory completeness and corruption tests. | +| `internal/compat/schema.go` | Read SQLite metadata and classify deterministic schema signatures. | +| `internal/compat/schema_test.go` | Shape, unsupported-lineage, and idempotency tests. | +| `internal/compat/testdata/release-schemas/manifest.json` | Hermetic v0.3.7–v3.2.5 release-to-fixture inventory and provenance checksums. | +| `internal/compat/testdata/release-schemas/*.sql` | Unique published V1–V13 shapes plus observed `source_metadata` variants. | +| `internal/storage/migration_plan.go` | Snapshot, execute named steps in one transaction, verify final shape, and reopen. | +| `internal/storage/migration_plan_test.go` | Lossless lineage, destructive snapshot, rollback, and final-shape tests. | +| `internal/storage/migrations.go` | Existing V1–V13 SQL bodies refactored into transaction-aware step helpers. | +| `internal/storage/storage.go` | Open a supported lineage through inspection and the safe migration executor. | + +### Task 1: Define the core schema contract and hermetic release inventory + +**Files:** +- Create: `internal/compat/types.go` +- Create: `internal/compat/catalog.go` +- Create: `internal/compat/catalog_test.go` +- Create: `internal/compat/testdata/release-schemas/manifest.json` +- Create: `internal/compat/testdata/release-schemas/v1.sql` +- Create: `internal/compat/testdata/release-schemas/v2.sql` +- Create: `internal/compat/testdata/release-schemas/v3.sql` +- Create: `internal/compat/testdata/release-schemas/v3-no-source-metadata.sql` +- Create: `internal/compat/testdata/release-schemas/v4.sql` +- Create: `internal/compat/testdata/release-schemas/v5-with-source-metadata.sql` +- Create: `internal/compat/testdata/release-schemas/v5-without-source-metadata.sql` +- Create: `internal/compat/testdata/release-schemas/v7.sql` +- Create: `internal/compat/testdata/release-schemas/v8.sql` +- Create: `internal/compat/testdata/release-schemas/v9.sql` +- Create: `internal/compat/testdata/release-schemas/v10.sql` +- Create: `internal/compat/testdata/release-schemas/v11.sql` +- Create: `internal/compat/testdata/release-schemas/v12.sql` +- Create: `internal/compat/testdata/release-schemas/v13.sql` + +**Interfaces:** +- Consumes: existing `*sql.DB`, whose `QueryContext(context.Context, string, ...any) (*sql.Rows, error)` and `QueryRowContext(context.Context, string, ...any) *sql.Row` satisfy `compat.Queryer`. +- Produces: `Code`, `Diagnostic`, `SchemaShape`, `MigrationStep`, `MigrationPlan`, `Queryer`, `Catalog`, and `func LoadCatalog() (Catalog, error)`. No manifest or recovery types are introduced. + +- [ ] **Step 1: Write the release inventory failure first** + +```go +func TestCheckedInReleaseSchemaManifestIsComplete(t *testing.T) { + catalog, err := LoadCatalog() + if err != nil { t.Fatal(err) } + if catalog.FirstGoRelease != "v0.3.7" || catalog.LatestGoRelease != "v3.2.5" { + t.Fatalf("catalog bounds = %s..%s", catalog.FirstGoRelease, catalog.LatestGoRelease) + } + seen := map[string]bool{} + for _, release := range catalog.Releases { + if release.Tag == "" || release.Fixture == "" || release.ProvenanceSHA256 == "" || seen[release.Tag] { + t.Fatalf("invalid release mapping: %+v", release) + } + seen[release.Tag] = true + if _, err := fs.Stat(releaseSchemaFS, "testdata/release-schemas/"+release.Fixture); err != nil { + t.Fatalf("fixture %q: %v", release.Fixture, err) + } + } + if !seen["v0.3.7"] || !seen["v3.2.5"] { t.Fatalf("release endpoints missing: %v", seen) } +} +``` + +- [ ] **Step 2: Run RED and confirm the missing catalog is the reason** + +Run: `go test ./internal/compat -run '^TestCheckedInReleaseSchemaManifestIsComplete$'` + +Expected: FAIL to compile with `undefined: LoadCatalog` or fail because `manifest.json` is absent; it must not fail from HOME, Git, or network access. + +- [ ] **Step 3: Add the minimal schema-only contract and embedded catalog** + +```go +type Code string + +const ( + CodeUnsupportedLineage Code = "unsupported_lineage" + CodeMigrationFailed Code = "migration_failed" + CodeIndexStale Code = "index_stale" +) + +type Diagnostic struct { + Code Code + Summary string + Continuation []string +} + +type SchemaShape struct { AppliedVersion int; Signature string } +type MigrationStep struct { Version int; Name string } +type MigrationPlan struct { From SchemaShape; Steps []MigrationStep } + +type Queryer interface { + QueryContext(context.Context, string, ...any) (*sql.Rows, error) + QueryRowContext(context.Context, string, ...any) *sql.Row +} +``` + +`Continuation` is present because later command policy consumes it, but Plan 1 never renders it and does not invent a `recover` argv. The catalog lists every published Go release tag individually and maps it to one exact fixture and SHA-256 provenance. Use the approved spans: V1 `v0.3.7–v0.3.10`, V2/V3 `v0.3.11–v1.3.5`, V4 `v1.4.0–v1.4.4`, V5 variants `v2.0.0–v2.1.0`, V7 `v2.2.0–v2.2.3`, V8 `v2.3.0`, V9 `v2.4.0–v2.5.0`, V10 `v2.6.0`, V11 `v2.7.0`, V12 `v2.8.0–v2.11.0`, and V13 `v2.12.0–v3.2.5`. + +Because local tags stop at v2.16.1, an explicitly authorized maintainer capture may run `gh release list --repo pablontiv/backscroll --limit 200 --json tagName,publishedAt` once while authoring the checked-in manifest. No test, build, or CI command runs it. + +- [ ] **Step 4: GREEN, then triangulate truncation and missing fixtures** + +Run: `go test ./internal/compat -run '^(TestCheckedInReleaseSchemaManifestIsComplete|TestReleaseSchemaManifestRejectsMissingFixture|TestReleaseSchemaManifestRejectsLatestBeforeV3_2_5)$'` + +Expected: PASS; tests replace the embedded FS with `fstest.MapFS` and reject a missing fixture and a latest bound below v3.2.5 without invoking Git or HTTP. + +- [ ] **Step 5: Refactor and run package tests** + +Run: `go test ./internal/compat` + +Expected: PASS with no network or environment dependency. + +- [ ] **Step 6: Commit the coherent inventory work unit** + +```bash +git add internal/compat/types.go internal/compat/catalog.go internal/compat/catalog_test.go internal/compat/testdata/release-schemas +git commit -m "feat(compat): add hermetic release schema inventory" +``` + +### Task 2: Inspect real SQLite shape and return typed migration decisions + +**Files:** +- Create: `internal/compat/schema.go` +- Create: `internal/compat/schema_test.go` +- Modify: `internal/compat/catalog.go` + +**Interfaces:** +- Consumes: `Queryer` and embedded `Catalog` from Task 1. +- Produces: `func InspectIndex(ctx context.Context, q Queryer) (MigrationPlan, *Diagnostic, error)` and `func VerifyCurrentShape(ctx context.Context, q Queryer) error`. + +- [ ] **Step 1: Write table-driven shape tests** + +```go +func TestInspectIndexUsesObservedShapeNotVersionAlone(t *testing.T) { + tests := []struct{ fixture, wantFirstStep string }{ + {"v5-with-source-metadata.sql", "V6 drop source_metadata when present"}, + {"v5-without-source-metadata.sql", "V7 reasoning triggers"}, + } + for _, tt := range tests { t.Run(tt.fixture, func(t *testing.T) { + db := openFixtureCopy(t, tt.fixture) + plan, diag, err := InspectIndex(context.Background(), db) + if err != nil || diag != nil { t.Fatalf("plan error=%v diagnostic=%+v", err, diag) } + if len(plan.Steps) == 0 || plan.Steps[0].Name != tt.wantFirstStep { t.Fatalf("steps=%+v", plan.Steps) } + }) } +} +``` + +Also add `TestInspectIndexCurrentShapeIsIdempotent` and `TestInspectIndexUnsupportedShapeReturnsInternalDiagnostic`. The unsupported-shape assertion requires `CodeUnsupportedLineage`, the observed signature in `Summary`, and an empty `Continuation`; Plan 3 supplies executable argv only when it activates command policy. + +- [ ] **Step 2: Run RED** + +Run: `go test ./internal/compat -run '^TestInspectIndex'` + +Expected: FAIL with `undefined: InspectIndex`. + +- [ ] **Step 3: Implement deterministic metadata inspection** + +Query `schema_migrations`, `sqlite_master`, `PRAGMA table_info()`, `PRAGMA index_list()`, `PRAGMA index_info()`, and relevant trigger SQL. Canonicalize by sorting records as `kind|table|name|columns|sql`, exclude volatile SQLite metadata, hash with SHA-256, and match only the checked-in catalog. + +```go +func InspectIndex(ctx context.Context, q Queryer) (MigrationPlan, *Diagnostic, error) { + shape, err := inspectShape(ctx, q) + if err != nil { return MigrationPlan{}, nil, fmt.Errorf("inspect schema: %w", err) } + lineage, ok := defaultCatalog.BySignature(shape.Signature) + if !ok { + return MigrationPlan{}, &Diagnostic{ + Code: CodeUnsupportedLineage, + Summary: fmt.Sprintf("unsupported index schema %s", shape.Signature), + }, nil + } + return MigrationPlan{From: shape, Steps: lineage.RemainingSteps()}, nil, nil +} +``` + +- [ ] **Step 4: GREEN and triangulate corrupt metadata** + +Run: `go test ./internal/compat -run '^(TestInspectIndex|TestVerifyCurrentShape)'` + +Expected: PASS; malformed SQLite returns a wrapped Go error, while a readable unknown schema returns `CodeUnsupportedLineage` without user-facing command text. + +- [ ] **Step 5: Refactor canonicalization and rerun the package** + +Run: `go test ./internal/compat` + +Expected: PASS; metadata order changes do not change signatures. + +- [ ] **Step 6: Commit the inspector** + +```bash +git add internal/compat/schema.go internal/compat/schema_test.go internal/compat/catalog.go +git commit -m "feat(compat): inspect observed index schema lineage" +``` + +### Task 3: Execute migration plans with snapshot, one transaction, and independent verification + +**Files:** +- Create: `internal/storage/migration_plan.go` +- Create: `internal/storage/migration_plan_test.go` +- Modify: `internal/storage/migrations.go:8-175` +- Modify: `internal/storage/storage.go:20-47` + +**Interfaces:** +- Consumes: `compat.InspectIndex`, `compat.MigrationPlan`, and existing V1–V13 SQL bodies in `migrations.go`. +- Produces: `func OpenCompatible(ctx context.Context, path string) (*Database, *compat.Diagnostic, error)`, `func (d *Database) ApplyMigrationPlan(ctx context.Context, path string, plan compat.MigrationPlan) error`, and `func SnapshotDatabase(ctx context.Context, srcPath string) (string, error)`. + +- [ ] **Step 1: Write lossless and rollback tests first** + +Add named tests `TestPublishedGoLineagesUpgradeLosslessly`, `TestHistoricalLineageWithoutSourceMetadataUpgradesLosslessly`, and `TestMigrationSnapshotAndRollbackOnDestructiveFailure`. Copy each SQL fixture into `t.TempDir()`, seed sentinel rows, and assert row identity/count plus FTS queryability after migration. Inject failure after the first destructive step and assert the original database remains at its pre-transaction shape and the sibling snapshot reopens read-only. + +- [ ] **Step 2: Run RED** + +Run: `go test ./internal/storage -run '^(TestPublishedGoLineagesUpgradeLosslessly|TestHistoricalLineageWithoutSourceMetadataUpgradesLosslessly|TestMigrationSnapshotAndRollbackOnDestructiveFailure)$'` + +Expected: FAIL because `Open` still runs migrations independently and V6 fails on the missing-column fixture. + +- [ ] **Step 3: Implement minimal migration execution** + +Refactor migration bodies into transaction-aware helpers with exact shape `func applyV6(ctx context.Context, tx *sql.Tx, from compat.SchemaShape) error`; preserve existing SQL and migration names. `ApplyMigrationPlan` creates and fsyncs a sibling snapshot before the first destructive step, begins one transaction, dispatches only known `(Version, Name)` pairs, calls `compat.VerifyCurrentShape(ctx, tx)`, and commits once. Reopen read-only and verify current shape after commit before returning success. + +```go +func OpenCompatible(ctx context.Context, path string) (*Database, *compat.Diagnostic, error) { + inspect, err := OpenReadOnly(path) + if errors.Is(err, fs.ErrNotExist) { + db, openErr := Open(path) + return db, nil, openErr + } + if err != nil { return nil, nil, err } + plan, diag, err := compat.InspectIndex(ctx, inspect.DB()) + _ = inspect.Close() + if err != nil || diag != nil { return nil, diag, err } + db, err := openWithoutSetup(path) + if err == nil && len(plan.Steps) > 0 { err = db.ApplyMigrationPlan(ctx, path, plan) } + return db, nil, err +} +``` + +`OpenCompatible` is a migration primitive only. No existing command is switched to it in Plan 1, so this slice cannot introduce a refusal with a nonexistent continuation. + +- [ ] **Step 4: GREEN and triangulate final-shape failure** + +Run: `go test ./internal/storage -run '^(TestPublishedGoLineagesUpgradeLosslessly|TestHistoricalLineageWithoutSourceMetadataUpgradesLosslessly|TestMigrationSnapshotAndRollbackOnDestructiveFailure|TestMigrationFinalShapeFailureRollsBack)$'` + +Expected: PASS; injected final-shape failure occurs before commit and leaves all seeded rows unchanged. + +- [ ] **Step 5: Refactor without widening compatibility scope** + +Run: `go test ./internal/storage` + +Expected: PASS; existing V8–V13 migration tests remain green. + +- [ ] **Step 6: Run the Plan 1 boundary and repository gates** + +Run: `go test ./internal/compat ./internal/storage -run '^(TestCheckedInReleaseSchemaManifestIsComplete|TestPublishedGoLineagesUpgradeLosslessly|TestHistoricalLineageWithoutSourceMetadataUpgradesLosslessly|TestMigrationSnapshotAndRollbackOnDestructiveFailure|TestMigrationFinalShapeFailureRollsBack)$'` + +Expected: PASS with no skipped tests and no Cobra/recovery dependency. + +Run: `just check` + +Expected: PASS. + +Run: `just test` + +Expected: PASS. + +Run: `just ci` + +Expected: PASS. + +- [ ] **Step 7: Record the safe intermediate boundary and commit** + +The implementation PR states: “Plan 1 provides inspected, shape-safe migration primitives only. It does not activate stale-index refusal, emit `recover`, or close issue #31. Plan 3 activates the policy after recovery exists; Plan 2 supplies the ingestion half required to close #31.” + +```bash +git add internal/storage/migration_plan.go internal/storage/migration_plan_test.go internal/storage/migrations.go internal/storage/storage.go +git commit -m "feat(storage): migrate supported lineages atomically" +``` diff --git a/docs/superpowers/plans/2026-08-18-manifest-source-ingestion.md b/docs/superpowers/plans/2026-08-18-manifest-source-ingestion.md new file mode 100644 index 0000000..fb35879 --- /dev/null +++ b/docs/superpowers/plans/2026-08-18-manifest-source-ingestion.md @@ -0,0 +1,345 @@ +# Manifest Source Ingestion Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make `*.inputs.toml` manifests the only external-source truth, preflight every active input before sync, and retrieve nested/new decision Markdown through a registered `markdown_sections` reader. + +**Architecture:** Extend the stateless `internal/compat` boundary with manifest types only when this plan consumes them, while keeping file discovery, decoding, and `SyncFiles` execution in their existing consumers. Register one normal `SessionReader` for Markdown sections and reuse `input_config.DiscoverFiles` plus `readers.Registry`. Plan 3’s already-active fail-closed policy guarantees that any manifest failure blocks cached index output. + +**Tech Stack:** Go 1.26.2, `pelletier/go-toml/v2`, existing `internal/input_config`, existing `internal/readers`, SHA-256 hashing, Cobra CLI integration tests, Just + +**Spec:** `docs/superpowers/specs/2026-08-18-systemic-index-compatibility-design.md` + +## Global Constraints + +- This is delivery Plan 2 but executes third: Plan 1 inspection/migration primitives → Plan 3 recovery and blocking-policy activation → Plan 2 manifest ingestion → Plan 4 shipped guidance validation. +- This plan depends on Plan 1’s core `compat.Diagnostic` and Plan 3’s active command policy, stale-index behavior, direct-read exemption, and executable `recover` continuation. It introduces `ManifestPlan`, `ResolvedInput`, and `ReaderLookup` only in Task 2 when preflight first consumes them. +- `*.inputs.toml` is the sole external-source truth. Non-empty legacy `[sources]` is rejected; no config mutation, manifest generation, warning-only path, or dual ingestion is allowed. +- Legacy rejection prints a complete exact manifest using the observed category and paths. +- Every active manifest is loaded and registry-resolved before discovery, hashing, parsing, or syncing any input. +- Invalid TOML, unsupported manifest version, duplicate input ID, missing required field, unknown decoder, invalid/inaccessible root, and unsafe pattern are visible blocking failures in deterministic manifest-file/input-ID order. +- An existing valid root with no matching files is a valid empty input. A missing or inaccessible declared root is invalid. +- `markdown_sections` is a normal `readers.SessionReader`; it uses existing manifest discovery and deterministic hashing, emits ordered eligible sections, and falls back to one whole-document record when no eligible heading exists. +- Parse all planned files before calling existing `storage.SyncFiles`; any failure propagates through Plan 3’s active stale policy and no indexed command serves cached rows. +- Direct `read` remains available and makes no index-freshness claim. +- Tests set `HOME` and `BACKSCROLL_CONFIG_DIR` to `t.TempDir()` and use no network, ambient tags, or user files. +- Follow strict RED → GREEN → TRIANGULATE → REFACTOR. Run focused tests before `just check`, `just test`, and `just ci`. +- Commit commands below are future execution instructions only. Do not stage or commit while writing or reviewing this plan. + +--- + +## File map + +| Path | Responsibility | +|---|---| +| `internal/compat/manifest.go` | Inspect raw `[sources]` and active definitions; return typed diagnostics or resolved input plans. | +| `internal/compat/manifest_test.go` | Exact conversion, precedence, duplicate/version/root/decoder preflight tests. | +| `internal/config/config.go` | Expose deterministic raw loaded config bytes without changing effective config behavior. | +| `internal/input_config/loader.go` | Load all definitions with file provenance before filtering active entries. | +| `internal/input_config/types.go` | Add manifest provenance and validation result types used by preflight. | +| `internal/input_config/discover.go` | Return declared-root errors instead of silently skipping invalid roots. | +| `internal/readers/markdown_sections.go` | Discover, hash, and parse Markdown sections through `SessionReader`. | +| `internal/readers/markdown_sections_test.go` | Ordered sections, heading levels, frontmatter, and whole-document fallback. | +| `internal/readers/reader.go` | Provide one production registry constructor containing all built-in readers. | +| `cmd/backscroll/sync_helpers.go` | Run all-active preflight, then discovery/parse, then one existing sync execution. | +| `cmd/backscroll/manifest_ingestion_test.go` | CLI-level preflight, no-partial-sync, legacy rejection, and decision retrieval. | +| `inputs/decisions.inputs.toml` | Active repaired decision preset using `markdown_sections` and real command instructions. | +| `tests/fixtures/decisions/**` | Nested/new Markdown fixtures for end-to-end retrieval. | + +### Task 1: Load manifest provenance and reject legacy `[sources]` exactly + +**Files:** +- Modify: `internal/compat/types.go` +- Create: `internal/compat/manifest.go` +- Create: `internal/compat/manifest_test.go` +- Modify: `internal/config/config.go:42-115` +- Modify: `internal/input_config/types.go:3-29` +- Modify: `internal/input_config/loader.go:27-68` + +**Interfaces:** +- Consumes: core `compat.Diagnostic`, existing `input_config.InputDefinition`, and TOML parser. +- Produces: `const compat.CodeLegacySources Code = "legacy_sources"`, `type input_config.LoadedDefinition struct { File string; Definition InputDefinition }`, `func input_config.LoadAllFromDir(dir string) ([]LoadedDefinition, error)`, `type config.LoadedFile struct { Path string; Data []byte }`, `func config.LoadedConfigFiles() ([]LoadedFile, error)`, and `func compat.InspectLegacyConfig(raw []byte) *Diagnostic`. + +- [ ] **Step 1: Write the exact legacy conversion failure test** + +```go +func TestLegacySourcesRejectedWithExactManifestExample(t *testing.T) { + raw := []byte("[sources]\ndecisions = [\"/work/project/docs/decisions\"]\n") + d := InspectLegacyConfig(raw) + if d == nil || d.Code != CodeLegacySources { t.Fatalf("diagnostic=%+v", d) } + for _, want := range []string{ + `id = "decisions"`, `source = "decision"`, + `roots = ["/work/project/docs/decisions"]`, `include = ["**/*.md"]`, + `format = "markdown_sections"`, + } { + if !strings.Contains(d.Summary, want) { t.Fatalf("summary missing %q: %s", want, d.Summary) } + } +} +``` + +At CLI level, snapshot config and database bytes before the call and assert neither changes. + +- [ ] **Step 2: Run RED** + +Run: `go test ./internal/compat -run '^TestLegacySourcesRejectedWithExactManifestExample$'` + +Expected: FAIL with `undefined: InspectLegacyConfig`. + +- [ ] **Step 3: Implement raw config inspection and provenance-preserving loads** + +Add only `CodeLegacySources` to `internal/compat/types.go` because this task is its first consumer. `LoadedConfigFiles` returns readable global then local config bytes in the same precedence order `config.Load` already applies. `LoadAllFromDir` sorts `*.inputs.toml` paths, requires `version == 1`, retains inactive entries for validation, and records each source file. `InspectLegacyConfig` parses only enough TOML to observe `sources`, orders categories as `backlog`, `decisions`, `ke`, `memories`, `rules`, `specs`, and returns one complete manifest block per non-empty category. Its continuation is `[]string{"config"}` because `backscroll config` is executable and exposes the manifest location; the summary itself carries the repair text. + +- [ ] **Step 4: GREEN and triangulate empty/invalid config** + +Run: `go test ./internal/compat ./internal/input_config ./internal/config -run '^(TestLegacySourcesRejectedWithExactManifestExample|TestEmptyLegacySourcesAllowed|TestLegacySourcesPreserveCategoryAndPathOrder|TestLoadAllFromDirRejectsUnsupportedVersion|TestLoadedConfigFilesFollowEffectivePrecedence)$'` + +Expected: PASS; empty `[sources]` returns nil, invalid TOML remains precedence level 1 as a Go parse error, and no test reads real HOME. + +- [ ] **Step 5: Refactor exact manifest formatting and run affected packages** + +Run: `go test ./internal/compat ./internal/input_config ./internal/config` + +Expected: PASS. + +- [ ] **Step 6: Commit legacy rejection and provenance as one review unit** + +```bash +git add internal/compat/types.go internal/compat/manifest.go internal/compat/manifest_test.go internal/config/config.go internal/input_config/types.go internal/input_config/loader.go +git commit -m "feat(inputs): reject legacy sources with exact manifests" +``` + +### Task 2: Preflight every active manifest before any discovery or sync + +**Files:** +- Modify: `internal/compat/types.go` +- Modify: `internal/compat/manifest.go` +- Modify: `internal/compat/manifest_test.go` +- Modify: `internal/input_config/discover.go:16-51` +- Modify: `internal/readers/reader.go:25-75` +- Modify: `cmd/backscroll/sync_helpers.go:50-128` +- Create: `cmd/backscroll/manifest_ingestion_test.go` + +**Interfaces:** +- Consumes: `input_config.LoadedDefinition`, existing `Registry.ForDef(InputDefinition) (SessionReader, error)`, and core diagnostics. +- Produces: constants `CodeInvalidManifest = "invalid_manifest"`, `CodeUnknownDecoder = "unknown_decoder"`, and `CodeInvalidRoot = "invalid_root"`; `type ReaderLookup interface { ForDef(input_config.InputDefinition) (readers.SessionReader, error) }`, `type ManifestPlan struct { Inputs []ResolvedInput }`, `type ResolvedInput struct { Definition input_config.InputDefinition; ReaderName string }`, `func InspectManifests(defs []input_config.LoadedDefinition, registry ReaderLookup) (ManifestPlan, *Diagnostic, error)`, `func readers.NewDefaultRegistry() *Registry`, and `func preflightInputs(cfg *config.Config) (compat.ManifestPlan, *compat.Diagnostic, error)`. + +- [ ] **Step 1: Write all-active preflight tests** + +Add `TestActiveManifestsPreflightBeforeSync` as a table with invalid TOML, unsupported version, duplicate ID, missing ID/source/root/include/format, unknown decoder, missing root, inaccessible root, and invalid glob. Place one valid input first and assert its file is absent from `indexed_files` after every failing case. + +- [ ] **Step 2: Run RED** + +Run: `go test ./cmd/backscroll -run '^TestActiveManifestsPreflightBeforeSync$'` + +Expected: FAIL because `maybeAutoSync` currently continues after `ForDef`, discovery, hash, and parse errors and may sync earlier inputs. + +- [ ] **Step 3: Introduce the manifest-only contract and implement deterministic preflight** + +Add the three Task 2 diagnostic codes, `ReaderLookup`, `ManifestPlan`, and `ResolvedInput` to `internal/compat/types.go` with the exact signatures in Interfaces; do not add recovery or orchestration fields. Then implement: + +```go +func InspectManifests(defs []input_config.LoadedDefinition, registry ReaderLookup) (ManifestPlan, *Diagnostic, error) { + seen := map[string]string{} + plan := ManifestPlan{} + for _, loaded := range defs { + def := loaded.Definition + if !def.Active { continue } + if prior, ok := seen[def.ID]; ok { + return ManifestPlan{}, invalidManifest(def.ID, fmt.Sprintf("duplicate id in %s and %s", prior, loaded.File)), nil + } + seen[def.ID] = loaded.File + reader, err := registry.ForDef(def) + if err != nil { return ManifestPlan{}, unknownDecoder(def.ID, def.Decode.Format), nil } + if d := inspectRootAndPatterns(def); d != nil { return ManifestPlan{}, d, nil } + plan.Inputs = append(plan.Inputs, ResolvedInput{Definition: def, ReaderName: reader.Name()}) + } + return plan, nil, nil +} +``` + +`NewDefaultRegistry` registers `OpenCodeReader`, `ClaudeReader`, `PiReader`, and Task 3’s `MarkdownSectionsReader` exactly once. `DiscoverFiles` returns errors for invalid/inaccessible declared roots and unsafe patterns instead of `continue`; valid empty roots still return an empty result. + +- [ ] **Step 4: GREEN and triangulate deterministic precedence** + +Run: `go test ./internal/compat ./internal/input_config ./cmd/backscroll -run '^(TestActiveManifestsPreflightBeforeSync|TestManifestDiagnosticsFollowFileAndInputOrder|TestExistingRootWithNoMatchesIsValid|TestMissingDeclaredRootBlocks)$'` + +Expected: PASS; the first diagnostic follows sorted manifest path then declaration order, and no `SyncFiles` call occurs before all active definitions pass. + +- [ ] **Step 5: Refactor registry setup and run affected packages** + +Run: `go test ./internal/readers ./internal/input_config ./internal/compat ./cmd/backscroll` + +Expected: PASS. + +- [ ] **Step 6: Commit all-input preflight with its command proof** + +```bash +git add internal/compat/types.go internal/compat/manifest.go internal/compat/manifest_test.go internal/input_config/discover.go internal/readers/reader.go cmd/backscroll/sync_helpers.go cmd/backscroll/manifest_ingestion_test.go +git commit -m "fix(inputs): preflight all active manifests before sync" +``` + +### Task 3: Implement `markdown_sections` as a normal reader + +**Files:** +- Create: `internal/readers/markdown_sections.go` +- Create: `internal/readers/markdown_sections_test.go` +- Modify: `internal/readers/reader.go:30-75` + +**Interfaces:** +- Consumes: `input_config.DiscoverFiles`, `hashfile.HashFile`, `models.ParsedFile`, and `models.Message`. +- Produces: `type MarkdownSectionsReader struct{}`, with exact `SessionReader` methods `Name() string`, `Discover(input_config.InputDefinition) ([]string, error)`, `Hash(string) (string, error)`, and `Parse(string, input_config.InputDefinition) (models.ParsedFile, error)`. + +- [ ] **Step 1: Write deterministic parser tests** + +```go +func TestMarkdownSectionsReaderSectionsAndWholeDocument(t *testing.T) { + tests := []struct{ name, body string; want []string }{ + {"ordered sections", "# ADR 1\nintro\n## Context\nctx\n## Decision\nuse sqlite\n", []string{"ADR 1\nintro", "Context\nctx", "Decision\nuse sqlite"}}, + {"no heading", "plain decision body\n", []string{"plain decision body"}}, + {"frontmatter", "---\nid: ADR-7\n---\n# Choice\nbody\n", []string{"Choice\nbody"}}, + } + // Parse a temp file and compare Records[i].Content in order. +} +``` + +Also assert `Name() == "markdown_sections"`, deterministic SHA-256 hash, `Role == "document"`, and non-empty stable UUID derived from `path + heading byte offset`. + +- [ ] **Step 2: Run RED** + +Run: `go test ./internal/readers -run '^TestMarkdownSectionsReader'` + +Expected: FAIL with `undefined: MarkdownSectionsReader`. + +- [ ] **Step 3: Implement the minimal reader using existing discovery** + +`Discover` delegates directly to `input_config.DiscoverFiles(def.Discover)`. `Hash` delegates to `hashfile.HashFile`. `Parse` normalizes CRLF to LF, strips only a leading YAML frontmatter block delimited by exact `---` lines, treats ATX headings `#` through `######` as section boundaries, preserves body order, trims outer whitespace, and emits one whole-document record only when no non-empty heading section exists. + +```go +func (r *MarkdownSectionsReader) Name() string { return "markdown_sections" } +func (r *MarkdownSectionsReader) Discover(def input_config.InputDefinition) ([]string, error) { + return input_config.DiscoverFiles(def.Discover) +} +func (r *MarkdownSectionsReader) Hash(path string) (string, error) { return hashfile.HashFile(path) } +``` + +- [ ] **Step 4: GREEN and triangulate nested discovery/new file behavior** + +Run: `go test ./internal/readers -run '^(TestMarkdownSectionsReaderSectionsAndWholeDocument|TestMarkdownSectionsReaderDiscoversNestedMarkdown|TestMarkdownSectionsReaderHashChangesForNewContent|TestMarkdownSectionsReaderIgnoresTemplateExcludes)$'` + +Expected: PASS; nested `**/*.md` discovery uses the existing pipeline and adding a file changes only the discovered set, not parser ordering for existing files. + +- [ ] **Step 5: Refactor section scanning and verify the interface** + +Run: `go test ./internal/readers` + +Expected: PASS, including `var _ SessionReader = (*MarkdownSectionsReader)(nil)`. + +- [ ] **Step 6: Commit the reader and registration** + +```bash +git add internal/readers/markdown_sections.go internal/readers/markdown_sections_test.go internal/readers/reader.go +git commit -m "feat(readers): decode markdown into ordered sections" +``` + +### Task 4: Parse every planned file before one sync and repair the decisions preset + +**Files:** +- Modify: `cmd/backscroll/sync_helpers.go:16-160` +- Modify: `cmd/backscroll/manifest_ingestion_test.go` +- Modify: `inputs/decisions.inputs.toml` +- Create: `tests/fixtures/decisions/root.md` +- Create: `tests/fixtures/decisions/nested/decision.md` + +**Interfaces:** +- Consumes: `preflightInputs`, `readers.NewDefaultRegistry`, existing `Database.SyncFiles([]storage.IndexedFile) error`. +- Produces: parse-complete `func collectIndexedFiles(plan compat.ManifestPlan, registry *readers.Registry, existingHashes map[string]string) ([]storage.IndexedFile, error)` and repaired decision preset instructions. + +- [ ] **Step 1: Write the end-to-end retrieval test first** + +`TestDecisionManifestMarkdownSectionsEndToEnd` installs an active copy of `inputs/decisions.inputs.toml` under temporary `BACKSCROLL_CONFIG_DIR`, rewrites its root to `tests/fixtures/decisions`, runs `search --text "nested-decision-sentinel" --all-projects --json`, adds `new/deeper/late.md`, runs the same command again, and asserts both records have source `decision`. It sets HOME/config/database to temporary paths. + +- [ ] **Step 2: Run RED** + +Run: `go test ./cmd/backscroll -run '^TestDecisionManifestMarkdownSectionsEndToEnd$'` + +Expected: FAIL because `markdown_sections` is not used by the current ad hoc registry and the preset says `backscroll sync`, which is not a real command. + +- [ ] **Step 3: Collect all parses before mutating and repair instructions** + +Split `maybeAutoSync` into: load raw config and reject legacy sources; load all manifests; preflight all active definitions; discover/hash/parse every changed reference into memory; return on any error; call `SyncFiles` only after the complete collection succeeds. Keep existing project identification, tags, stale extraction, template, and correction derivation behavior after the preflight boundary. + +Change the preset comment from `Then: backscroll sync` to executable instructions: + +```text +# Verify: backscroll config +# Index and query: backscroll search --text "decision" --source decision --all-projects +``` + +Keep `format = "markdown_sections"` and activate the installed test copy, not necessarily the repository example, inside tests. + +- [ ] **Step 4: GREEN and triangulate parse failure/no partial refresh** + +Run: `go test ./cmd/backscroll -run '^(TestDecisionManifestMarkdownSectionsEndToEnd|TestParseFailurePreventsEveryInputSync|TestDiscoveryFailurePreventsCachedSearch|TestDirectReadRemainsAvailableButClaimsNoIndexFreshness)$'` + +Expected: PASS; introducing one unreadable Markdown file leaves the preexisting database byte/row state unchanged and blocks the indexed query. + +- [ ] **Step 5: Refactor collection into focused helpers and run affected packages** + +Run: `go test ./internal/input_config ./internal/readers ./internal/compat ./cmd/backscroll` + +Expected: PASS. + +- [ ] **Step 6: Commit the ingestion transaction boundary and preset** + +```bash +git add cmd/backscroll/sync_helpers.go cmd/backscroll/manifest_ingestion_test.go inputs/decisions.inputs.toml tests/fixtures/decisions +git commit -m "feat(inputs): ingest decision markdown after complete preflight" +``` + +### Task 5: Prove the ingestion half of #31 and close #33 + +**Files:** +- Modify: `internal/compat/manifest_test.go` +- Modify: `internal/readers/markdown_sections_test.go` +- Modify: `cmd/backscroll/manifest_ingestion_test.go` + +**Interfaces:** +- Consumes: all Plan 2 behavior plus Plan 3’s active stale-index policy and executable continuation contract. +- Produces: complete #31 ingestion-half and #33 closure evidence. + +- [ ] **Step 1: Run the named ingestion evidence** + +Run: `go test ./internal/compat ./internal/readers ./cmd/backscroll -run '^(TestLegacySourcesRejectedWithExactManifestExample|TestActiveManifestsPreflightBeforeSync|TestMarkdownSectionsReaderSectionsAndWholeDocument|TestDecisionManifestMarkdownSectionsEndToEnd)$'` + +Expected: PASS with hermetic HOME/config/database paths. + +- [ ] **Step 2: Run the complete #31 gate across Plans 1 and 2** + +Run: `go test ./internal/compat ./internal/storage ./internal/readers ./cmd/backscroll -run '^(TestCheckedInReleaseSchemaManifestIsComplete|TestPublishedGoLineagesUpgradeLosslessly|TestHistoricalLineageWithoutSourceMetadataUpgradesLosslessly|TestMigrationSnapshotAndRollbackOnDestructiveFailure|TestStaleIndexBlocksIndexBackedCommands|TestDirectReadRemainsAvailableButClaimsNoIndexFreshness|TestBlockingDiagnosticsHaveExecutableContinuations|TestLegacySourcesRejectedWithExactManifestExample|TestActiveManifestsPreflightBeforeSync|TestDecisionManifestMarkdownSectionsEndToEnd)$'` + +Expected: PASS with no skipped tests. Plan 2 never executes before Plan 3, so `recover` is registered and functional at this boundary. + +- [ ] **Step 3: Run repository gates** + +Run: `just check` + +Expected: PASS. + +Run: `just test` + +Expected: PASS. + +Run: `just ci` + +Expected: PASS. + +- [ ] **Step 4: Record issue closure evidence exactly** + +The implementation PR states: #33 closes only because `TestActiveManifestsPreflightBeforeSync` and `TestDecisionManifestMarkdownSectionsEndToEnd` pass; existing glob tests alone are insufficient. #31 closes only when every named Plan 1 migration test, every Plan 3 blocking/direct-read/executable-continuation test, and the three ingestion tests pass with no skipped case. + +- [ ] **Step 5: Commit only if closure assertions changed** + +```bash +git add internal/compat/manifest_test.go internal/readers/markdown_sections_test.go cmd/backscroll/manifest_ingestion_test.go +git commit -m "test(inputs): prove manifest-only decision ingestion" +``` diff --git a/docs/superpowers/plans/2026-08-18-shipped-guidance-integrity.md b/docs/superpowers/plans/2026-08-18-shipped-guidance-integrity.md new file mode 100644 index 0000000..75e9a37 --- /dev/null +++ b/docs/superpowers/plans/2026-08-18-shipped-guidance-integrity.md @@ -0,0 +1,338 @@ +# Shipped Guidance Integrity Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Ensure every `backscroll ` invocation in shipped skills, presets, and distributed documentation resolves and can be exercised safely through the real Cobra tree, while adding four mandatory search-discipline guardrails. + +**Architecture:** Build a test-only asset scanner that discovers shipped consumer assets by distribution roots, extracts shell-like Backscroll argv with source locations, classifies examples and destructive invocations, resolves every command/flag against `buildRootCmd`, and executes only through a hermetic safe harness. The validator derives command truth from Cobra and never maintains a command-name allowlist; production compatibility code remains untouched. + +**Tech Stack:** Go 1.26.2, Cobra 1.10.2, standard-library Markdown/TOML line scanning and argv lexer, temporary filesystem/config/database harness, table-driven Go tests, Just + +**Spec:** `docs/superpowers/specs/2026-08-18-systemic-index-compatibility-design.md` + +## Global Constraints + +- This plan runs last in the exact chain Plan 1 → Plan 3 → Plan 2 → Plan 4. It depends on Plan 3’s final command names/effects and Plan 2’s repaired manifest presets. +- Scan consumer `.md`, `.toml`, and `.sh` assets under `.claude/skills/**`, `inputs/*.toml`, root `README.md`, and distributed `docs/**/*.md`. Exclude only implementation-history trees `docs/roadmap/**`, `docs/research/**`, and `docs/superpowers/**`; these are contributor records, not shipped operating guidance. `.claude/skills/backscroll-doctor/assets/gather.sh` is shipped and must be discovered. +- Extract `backscroll ` invocations from prose, fenced code, inline code, and TOML comments. Report asset path, 1-based line, parsed argv, and Cobra/execution failure. +- Do not use a hand-maintained command or flag allowlist. Resolve against `buildRootCmd` and each command’s actual Cobra flags. +- Classify angle-bracket/metavariable examples explicitly and replace them with safe fixture values before execution. Never silently skip them after syntax resolution. +- Classify command effects from a canonical Cobra annotation on every root command constructor, not from verb strings. Missing or unknown effects fail closed before execution; there is no default effect. Exercise write/replace commands against disposable temporary data only. +- Never execute shell operators, substitutions, pipelines, redirects, external commands, or arbitrary prose. The scanner validates one parsed Backscroll invocation at a time. +- Every harness execution sets HOME, BACKSCROLL_CONFIG_DIR, and BACKSCROLL_DATABASE_PATH to `t.TempDir()` paths, disables ambient input discovery by supplying temporary manifests, and uses the dev version path so autoupdate performs no network access. +- Four search guardrails are mandatory: inspect ranked hits before concluding; use vocabulary from retrieved artifacts in follow-up queries; treat malformed/failed calls as tool failures rather than evidence of absence; reproduce suspected indexing gaps with a minimal fixture before claiming a gap. +- Issue #30 closes only when discovery, effect completeness/refusal, `TestAllShippedBackscrollCommandsAreExecutable`, and `TestShippedSearchGuidanceGuardrails` pass. The stale #33 preset is closed by Plan 2’s ingestion tests plus this plan’s command validation. +- Follow strict RED → GREEN → TRIANGULATE → REFACTOR. Run focused tests before `just check`, `just test`, and `just ci`. +- Commit commands below are future implementation instructions only. Do not stage or commit during planning. + +--- + +## File map + +| Path | Responsibility | +|---|---| +| `cmd/backscroll/shipped_assets_test.go` | Discover consumer assets, extract located invocations, resolve Cobra syntax, substitute fixtures, and execute safely. | +| `cmd/backscroll/shipped_guidance_test.go` | Assert the four semantic search guardrails across the primary Backscroll skill. | +| `cmd/backscroll/main.go` | Define the canonical effect annotation key/values, annotate the root, and register only explicitly annotated child commands. | +| `cmd/backscroll/{search,read,list,patterns,rebuild,purge,validate,status,config,annotate,recover}.go` | Declare each command’s maximum asset-execution effect explicitly at its canonical constructor. | +| `.claude/skills/backscroll/SKILL.md` | Correct commands and add the four search-discipline guardrails. | +| `.claude/skills/backscroll/ref-context-mode.md` | Replace stale flags/commands with real current invocations. | +| `.claude/skills/backscroll-doctor/SKILL.md` | Keep diagnostic invocations executable and evidence-disciplined. | +| `inputs/decisions.inputs.toml` | Validate Plan 2’s repaired command comments. | +| `README.md` and `docs/**/*.md` | Repair every scanner-reported shipped invocation while preserving intended workflow. | + +### Task 1: Discover shipped assets and extract located Backscroll argv + +**Files:** +- Create: `cmd/backscroll/shipped_assets_test.go` + +**Current candidate asset inventory:** +- `.claude/skills/backscroll/SKILL.md` +- `.claude/skills/backscroll/ref-context-mode.md` +- `.claude/skills/backscroll-doctor/SKILL.md` +- `.claude/skills/backscroll-doctor/assets/gather.sh` +- `inputs/categories.toml` +- `inputs/claude.inputs.toml` +- `inputs/decisions.inputs.toml` +- `inputs/opencode.inputs.toml` +- `inputs/pi.inputs.toml` +- `README.md` +- `docs/audit-integration.md` +- `docs/backlog.md` +- `docs/configuration.md` +- `docs/eval/README.md` +- `docs/eval/corrections-calibration.md` +- `docs/eval/corrections-labeling-2026-07-20.md` +- `docs/eval/queries.toml` +- `docs/input-contract.md` +- `docs/intention-agentic-input-definitions.md` +- `docs/patterns.md` +- `docs/read.md` +- `docs/search.md` +- `docs/sync.md` + +This inventory makes the current scan surface reviewable; discovery remains authoritative and catches future assets automatically. A valid discovered file is not modified unless its invocation fails extraction, Cobra resolution, safe execution, or the required guidance checks. + +**Interfaces:** +- Consumes: repository filesystem rooted deterministically from `runtime.Caller(0)` at `cmd/backscroll/shipped_assets_test.go` and walking `../..`; asset content is read with `os.DirFS` and tests do not require Git metadata. +- Produces: `type assetInvocation struct { Path string; Line int; Raw string; Argv []string; HasMetavariables bool; HasShellSyntax bool }`, `func shippedConsumerAssets(root string) ([]string, error)`, and `func extractBackscrollInvocations(path string, data []byte) ([]assetInvocation, error)`. + +- [ ] **Step 1: Write extraction tests covering prose, fences, TOML, and malformed calls** + +```go +func TestExtractBackscrollInvocations(t *testing.T) { + data := []byte("text `backscroll status`\n```bash\nbackscroll search --text \"two words\" --all-projects\n```\n# Then: backscroll recover --from --dry-run\n") + got, err := extractBackscrollInvocations("asset.md", data) + if err != nil { t.Fatal(err) } + if len(got) != 3 { t.Fatalf("invocations=%+v", got) } + if got[1].Line != 3 || !reflect.DeepEqual(got[1].Argv, []string{"search", "--text", "two words", "--all-projects"}) { + t.Fatalf("second=%+v", got[1]) + } + if !got[2].HasMetavariables { t.Fatalf("expected metavariable: %+v", got[2]) } +} +``` + +Add cases for single/double quotes, escaped spaces, comments after argv, multiple invocations on separate lines, unclosed quotes, `$()`, pipes, redirects, `&&`, and ellipsis. Add a shell-asset case with the exact canonical wrapper declaration `BS="${BACKSCROLL_BIN:-backscroll}"` followed by `"$BS" search "query" --all-projects --json | jq ...`; assert the scanner extracts only `search query --all-projects --json` and never executes the pipeline. + +- [ ] **Step 2: Run RED** + +Run: `go test ./cmd/backscroll -run '^TestExtractBackscrollInvocations$'` + +Expected: FAIL with `undefined: extractBackscrollInvocations`. + +- [ ] **Step 3: Implement deterministic discovery and a narrow argv lexer** + +`shippedConsumerAssets` walks the exact roots in Global Constraints, accepts `.md`, `.toml`, and `.sh` consumer assets, sorts slash-normalized paths, and applies only the three documented implementation-history exclusions. It must discover files rather than enumerate individual paths. The discovery test asserts every path in the current candidate inventory, including `gather.sh`, is present. + +The lexer starts at a literal command-position `backscroll`. For `.sh` only, it also recognizes the exact repository wrapper assignment `BS="${BACKSCROLL_BIN:-backscroll}"` and command-position `"$BS"`; it does not evaluate arbitrary variables or shell expansions. It parses quoting/escapes into argv, stops the Backscroll invocation before a shell operator, marks shell metacharacters without executing them, and returns a located parse error for malformed quotes. Quoted prose such as the query string `"backscroll search"` is not an invocation. It never invokes a shell. + +- [ ] **Step 4: GREEN and triangulate complete asset-root coverage** + +Run: `go test ./cmd/backscroll -run '^(TestExtractBackscrollInvocations|TestShippedConsumerAssetDiscovery)$'` + +Expected: PASS; discovery includes the complete current candidate inventory and excludes only `docs/roadmap/**`, `docs/research/**`, and `docs/superpowers/**`. + +- [ ] **Step 5: Refactor lexer state names and rerun focused tests** + +Run: `go test ./cmd/backscroll -run '^(TestExtractBackscrollInvocations|TestShippedConsumerAssetDiscovery)$'` + +Expected: PASS. + +- [ ] **Step 6: Commit the scanner as one reviewable test utility** + +```bash +git add cmd/backscroll/shipped_assets_test.go +git commit -m "test(guidance): extract commands from shipped assets" +``` + +### Task 2: Resolve every invocation against real Cobra and execute through a safe harness + +**Files:** +- Modify: `cmd/backscroll/shipped_assets_test.go` +- Modify: `cmd/backscroll/main.go:59-86` +- Modify: `cmd/backscroll/search.go:18-96` +- Modify: `cmd/backscroll/read.go:12-47` +- Modify: `cmd/backscroll/list.go:14-60` +- Modify: `cmd/backscroll/patterns.go:18-85` +- Modify: `cmd/backscroll/rebuild.go:15-29` +- Modify: `cmd/backscroll/purge.go:13-32` +- Modify: `cmd/backscroll/validate.go:14-35` +- Modify: `cmd/backscroll/status.go:15-42` +- Modify: `cmd/backscroll/config.go:14-42` +- Modify: `cmd/backscroll/annotate.go:13-46` +- Modify: `cmd/backscroll/recover.go` (created by Plan 3) + +**Interfaces:** +- Consumes: `buildRootCmd(io.Writer, io.Writer) *cobra.Command` and discovered `assetInvocation` values. +- Produces: constants `assetEffectAnnotation = "backscroll.io/effect"`, `assetEffectRead = "read"`, `assetEffectWrite = "write"`, and `assetEffectReplace = "replace"`; `func declaredAssetEffect(cmd *cobra.Command) (string, error)`; `func resolveInvocation(root *cobra.Command, argv []string) (*cobra.Command, error)`; `func materializeExample(inv assetInvocation, env safeAssetEnv) ([]string, error)`; and `func executeInvocation(inv assetInvocation) error`. + +- [ ] **Step 1: Write completeness and focused real-Cobra harness tests** + +Add `TestEveryRootCommandDeclaresAssetEffect`. Build `buildRootCmd`, inspect the root and every direct child, and fail with command path plus raw annotation when the key is missing, empty, or not exactly `read`, `write`, or `replace`. Add a synthetic unannotated command and an unknown-value command to prove `declaredAssetEffect` rejects both. + +```go +func TestAssetInvocationResolvesAgainstRealCobra(t *testing.T) { + tests := []struct{ raw string; wantErr bool }{ + {"backscroll status --json --indexed-only", false}, + {"backscroll search --text sentinel --all-projects", false}, + {"backscroll recover --from --dry-run", false}, + {"backscroll removed-command", true}, + {"backscroll search --removed-flag", true}, + } + for _, tt := range tests { + inv := mustExtractOne(t, tt.raw) + err := executeInvocation(inv) + if (err != nil) != tt.wantErr { t.Fatalf("%q error=%v", tt.raw, err) } + } +} +``` + +- [ ] **Step 2: Run RED** + +Run: `go test ./cmd/backscroll -run '^(TestEveryRootCommandDeclaresAssetEffect|TestAssetInvocationResolvesAgainstRealCobra)$'` + +Expected: FAIL because constructors do not declare effects and `executeInvocation` is undefined. + +- [ ] **Step 3: Implement syntax resolution, fixture substitution, and fail-closed effects** + +Build a fresh `buildRootCmd` for each invocation. Resolve command and flags through Cobra’s `Find`, `Args`, and `Flag` definitions; do not compare the verb to a list. Define the annotation vocabulary once in `main.go` and set it in every constructor listed in Files: + +- root, `read`, `validate`, `status`, and `config`: `read`; +- `search`, `list`, and `patterns`: `write`, because their normal path may auto-sync before reading; +- `rebuild`, `purge`, and `annotate`: `write`; +- `recover`: `replace`. + +`declaredAssetEffect` returns an error for missing, empty, or unknown annotation values. `executeInvocation` calls it after Cobra resolution and before fixture setup or command execution; no branch defaults to `read`. + +`materializeExample` replaces recognized metavariable shapes by semantic flag position: query/name text → `sentinel`; project → `all-projects` fixture ID; file/path → a temp JSONL fixture; date → `2030-01-01`; UUID → seeded temp record UUID; stranded database → a temp recovery fixture. Unknown metavariable shapes fail with path/line rather than skip. Shell syntax is accepted only when the extracted Backscroll argv is complete before the operator; never execute the operator or neighboring command. + +`executeInvocation` sets temp HOME/config/database, installs minimal valid manifests, seeds disposable state required by the command, sets `version = "dev"`, executes Cobra directly, and accepts domain errors only after command/flag/argument resolution when the example intentionally references absent user data. Write/replace annotations require disposable seeded paths and a postcondition that no path outside the temp root changed. + +- [ ] **Step 4: GREEN and triangulate destructive/example safety** + +Run: `go test ./cmd/backscroll -run '^(TestEveryRootCommandDeclaresAssetEffect|TestAssetInvocationResolvesAgainstRealCobra|TestAssetHarnessRejectsMissingOrUnknownEffect|TestAssetHarnessNeverExecutesShellSyntax|TestAssetHarnessContainsWriteCommandsToTempRoot|TestAssetHarnessRejectsUnknownMetavariable)$'` + +Expected: PASS; every root constructor is explicit, missing/unknown effects refuse execution, write/replace examples mutate only temporary state, unknown example variables remain hard failures, and removed commands/flags fail through Cobra rather than an allowlist. + +- [ ] **Step 5: Refactor harness setup and run the CLI package** + +Run: `go test ./cmd/backscroll` + +Expected: PASS with no network or real HOME/config access. + +- [ ] **Step 6: Commit the real-Cobra safety harness** + +```bash +git add cmd/backscroll/shipped_assets_test.go cmd/backscroll/main.go cmd/backscroll/search.go cmd/backscroll/read.go cmd/backscroll/list.go cmd/backscroll/patterns.go cmd/backscroll/rebuild.go cmd/backscroll/purge.go cmd/backscroll/validate.go cmd/backscroll/status.go cmd/backscroll/config.go cmd/backscroll/annotate.go cmd/backscroll/recover.go +git commit -m "test(guidance): execute shipped commands through cobra" +``` + +### Task 3: Repair shipped commands and add the four search guardrails + +**Files:** +- Create: `cmd/backscroll/shipped_guidance_test.go` +- Modify: `.claude/skills/backscroll/SKILL.md` +- Modify: `.claude/skills/backscroll/ref-context-mode.md` +- Modify: `inputs/decisions.inputs.toml` +- Modify: `inputs/opencode.inputs.toml` +- Modify: `inputs/pi.inputs.toml` +- Modify: `README.md` +- Modify: `docs/audit-integration.md` +- Modify: `docs/configuration.md` +- Modify: `docs/eval/README.md` +- Modify: `docs/eval/corrections-calibration.md` +- Modify: `docs/input-contract.md` +- Modify: `docs/intention-agentic-input-definitions.md` +- Modify: `docs/patterns.md` +- Modify: `docs/read.md` +- Modify: `docs/sync.md` + +These are the current known stale scanner failures. Do not modify valid discovered assets merely because they are in the candidate inventory. If discovery finds a future failing path not listed here, stop and add that exact path to the task’s Files list before editing it; never use a wildcard staging command or a generic “all discovered files” instruction. + +**Interfaces:** +- Consumes: final command/flag contracts from Plan 3, repaired presets from Plan 2, and the failing path/line/argv report from Task 2. +- Produces: executable shipped guidance and `func searchGuidanceGuardrails(data []byte) []string`, where an empty result means all four semantic requirements are present. + +- [ ] **Step 1: Write the all-assets and semantic guardrail assertions first** + +```go +func TestAllShippedBackscrollCommandsAreExecutable(t *testing.T) { + root := repositoryRoot(t) + assets, err := shippedConsumerAssets(root) + if err != nil { t.Fatal(err) } + for _, path := range assets { + data, err := os.ReadFile(filepath.Join(root, path)) + if err != nil { t.Fatal(err) } + invocations, err := extractBackscrollInvocations(path, data) + if err != nil { t.Fatal(err) } + for _, inv := range invocations { t.Run(fmt.Sprintf("%s:%d", inv.Path, inv.Line), func(t *testing.T) { + if err := executeInvocation(inv); err != nil { t.Fatalf("%s:%d argv=%q: %v", inv.Path, inv.Line, inv.Argv, err) } + }) } + } +} +func TestShippedSearchGuidanceGuardrails(t *testing.T) { + data, err := os.ReadFile(filepath.Join(repositoryRoot(t), ".claude/skills/backscroll/SKILL.md")) + if err != nil { t.Fatal(err) } + if missing := searchGuidanceGuardrails(data); len(missing) != 0 { t.Fatalf("missing search guardrails: %v", missing) } +} +``` + +Implement the guardrail helper as four independent phrase-family checks so harmless wording changes are allowed but each required behavior remains explicit: ranked-hit inspection before conclusion; follow-up query vocabulary from retrieved artifacts; malformed/failed call is not absence evidence; minimal fixture reproduction before index-gap claim. + +- [ ] **Step 2: Run RED and capture real drift** + +Run: `go test ./cmd/backscroll -run '^(TestAllShippedBackscrollCommandsAreExecutable|TestShippedSearchGuidanceGuardrails)$'` + +Expected: FAIL with located stale invocations such as `inputs/decisions.inputs.toml`’s historical `backscroll sync`, `.claude/skills/backscroll/SKILL.md`’s `backscroll version`, and `ref-context-mode.md`’s removed `--input` flag, plus absent guardrails. Preserve path/line/argv in every command failure. + +- [ ] **Step 3: Repair guidance from scanner output and add explicit guardrails** + +For each located failure in the Files list, use `backscroll --help` from the current Cobra tree to select the real replacement. Required known repairs include: use `backscroll --version`, not `backscroll version`; replace removed `--input` filters with current `--source` or `--source-path` semantics; replace removed `sync`, `inputs`, `sessions`, `events`, `resume`, and `topics` workflows with current `search`, `list`, `config`, `status`, `validate`, `read`, or `rebuild` commands according to the surrounding intent; replace unsupported audit/example flags rather than teaching the harness to accept domain failures; retain Plan 2’s decisions preset `config` plus `search --source decision` instructions. + +Add a concise “Evidence discipline” section to the Backscroll skill with all four imperative rules. Do not imply a zero-result search proves absence, a failed command proves an index gap, or direct `read` proves index freshness. + +- [ ] **Step 4: GREEN and triangulate every shipped asset** + +Run: `go test ./cmd/backscroll -run '^(TestShippedSearchGuidanceGuardrails|TestAllShippedBackscrollCommandsAreExecutable)$'` + +Expected: PASS with zero stale command/flag reports. + +- [ ] **Step 5: Refactor prose for scanability and rerun tests** + +Keep commands adjacent to their purpose, put the guardrails before troubleshooting, and remove contradictory stale instructions rather than appending caveats. Run: `go test ./cmd/backscroll -run '^(TestShippedSearchGuidanceGuardrails|TestAllShippedBackscrollCommandsAreExecutable)$'` + +Expected: PASS. + +- [ ] **Step 6: Commit each coherent guidance repair with its validator** + +```bash +git add .claude/skills/backscroll/SKILL.md .claude/skills/backscroll/ref-context-mode.md inputs/decisions.inputs.toml inputs/opencode.inputs.toml inputs/pi.inputs.toml README.md docs/audit-integration.md docs/configuration.md docs/eval/README.md docs/eval/corrections-calibration.md docs/input-contract.md docs/intention-agentic-input-definitions.md docs/patterns.md docs/read.md docs/sync.md cmd/backscroll/shipped_guidance_test.go +git commit -m "docs: align shipped guidance with executable commands" +``` + +### Task 4: Close issue #30 and the stale-preset portion of #33 + +**Files:** +- Modify: `cmd/backscroll/shipped_assets_test.go` +- Modify: `cmd/backscroll/shipped_guidance_test.go` + +**Interfaces:** +- Consumes: all Plan 4 validation plus Plan 2 repaired preset and Plan 3 `recover` registration. +- Produces: complete #30 closure evidence and final command-integrity evidence for #33. + +- [ ] **Step 1: Run the named discovery, effect, execution, and guidance closure tests** + +Run: `go test ./cmd/backscroll -run '^(TestShippedConsumerAssetDiscovery|TestEveryRootCommandDeclaresAssetEffect|TestAssetHarnessRejectsMissingOrUnknownEffect|TestAllShippedBackscrollCommandsAreExecutable|TestShippedSearchGuidanceGuardrails)$'` + +Expected: PASS; discovery includes every current candidate and `gather.sh`; missing/unknown effects refuse execution; invocation failures include asset path, 1-based line, parsed argv, and Cobra or safe-harness cause. + +- [ ] **Step 2: Triangulate examples and destructive invocations** + +Run: `go test ./cmd/backscroll -run '^(TestEveryRootCommandDeclaresAssetEffect|TestAssetHarnessRejectsMissingOrUnknownEffect|TestAssetHarnessNeverExecutesShellSyntax|TestAssetHarnessContainsWriteCommandsToTempRoot|TestAssetHarnessRejectsUnknownMetavariable|TestAllShippedBackscrollCommandsAreExecutable)$'` + +Expected: PASS; no invocation is accepted by string matching alone, no unknown metavariable is skipped, missing/unknown effects refuse execution, and no write/replace command escapes the disposable root. + +- [ ] **Step 3: Run repository gates** + +Run: `just check` + +Expected: PASS. + +Run: `just test` + +Expected: PASS. + +Run: `just ci` + +Expected: PASS. + +- [ ] **Step 4: Record exact issue closure evidence** + +The implementation PR states: #30 closes because the five named discovery/effect/execution/guidance tests pass, every current root constructor declares a known effect, missing/unknown effects fail closed, every invocation across every discovered shipped consumer asset resolves against `buildRootCmd`, safe execution uses hermetic fixture substitution, and all four guardrails are present. The stale-preset portion of #33 closes because `inputs/decisions.inputs.toml` contains only executable commands; #33 itself also requires Plan 2’s `TestActiveManifestsPreflightBeforeSync` and `TestDecisionManifestMarkdownSectionsEndToEnd`. + +- [ ] **Step 5: Commit only if closure assertions changed** + +```bash +git add cmd/backscroll/shipped_assets_test.go cmd/backscroll/shipped_guidance_test.go +git commit -m "test(guidance): prove shipped command integrity" +``` diff --git a/docs/superpowers/plans/2026-08-18-stranded-database-recovery.md b/docs/superpowers/plans/2026-08-18-stranded-database-recovery.md new file mode 100644 index 0000000..b80b603 --- /dev/null +++ b/docs/superpowers/plans/2026-08-18-stranded-database-recovery.md @@ -0,0 +1,592 @@ +# Stranded Database Recovery Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add `backscroll recover --from [--dry-run]` to replace the configured active database with the verified union of its active records and one stranded historical database without mutating either input during planning. + +**Architecture:** Extend Plan 1’s schema-only compatibility boundary with the recovery types and canonical record representation needed by this consumer, then build one complete active-first union plan. A separate recovery consumer creates and verifies a fresh current-schema sibling database, preserves the original active database, and performs same-filesystem atomic replacement. Only after `recover` is registered and functional does this plan activate one shared fail-closed index policy and user-facing executable continuations. + +**Tech Stack:** Go 1.26.2, `database/sql`, `modernc.org/sqlite`, `google/uuid`, Cobra 1.10.2, SHA-256, filesystem sync/rename primitives, table-driven Go integration tests, Just + +**Spec:** `docs/superpowers/specs/2026-08-18-systemic-index-compatibility-design.md` + +## Global Constraints + +- This is delivery Plan 3 but executes second: Plan 1 inspection/migration primitives → Plan 3 recovery and blocking-policy activation → Plan 2 manifest ingestion → Plan 4 shipped guidance validation. +- This plan depends only on Plan 1’s `compat.InspectIndex`, `compat.SchemaShape`, schema diagnostics, and current-shape verification. Plan 3 itself introduces recovery types and the shared canonical `models.IndexedRecord` only when recovery consumes them. +- The only command is `backscroll recover --from [--dry-run]`; destination is the configured active database. +- Do not add `--into`, `--force`, `--skip`, `--skip-conflicts`, `--merge`, `--partial`, multiple `--from` values, cross-machine transfer, synchronization, replication, or conflict resolution. +- Resolve active and stranded paths before opening. If both resolve to the same path, account for one logical input once while still using fresh destination and backup on apply. +- Open every distinct input with SQLite `mode=ro`; never migrate, vacuum, journal-mode change, or otherwise mutate either planning input. Stranded bytes and sidecar metadata remain immutable on dry-run, success, and failure. +- Identity is valid UUID first; only an empty UUID falls back to exact stored `(source_path, ordinal)` with non-negative ordinal. Payload hash proves equivalence only and never creates identity. +- Collapse only same-identity/equivalent-payload duplicates. Same identity/different payload or any uninterpretable row aborts the entire plan and apply. +- Account for every row from both inputs as importable, exact duplicate, conflicting, or uninterpretable before any write. +- Dry-run and apply call the identical planner; dry-run creates no destination, temporary database, backup, journal, or other file. +- Apply creates a fresh current-schema temporary database beside active, imports and verifies in one transaction, closes handles, independently reopens/verifies, preserves active as a unique backup, atomically replaces on the same filesystem, and fsyncs the directory after backup and replacement. +- A failed recovery never reports success and leaves the original active path intact or a restorable backup with an explicit diagnostic. Never delete the backup automatically. +- Stale-index refusal, cached-fallback removal, indexed-consumer changes, diagnostic command behavior, direct-read exemption tests, and executable-continuation tests activate only after `recover` is registered and functional in this plan. +- Every user-facing blocking diagnostic has non-empty argv that resolves and executes through the real Cobra tree; no test is skipped or deferred at any plan boundary. +- `backscroll read` remains behaviorally unchanged and bypasses index policy because it reads the supplied file directly; test that behavior without modifying or staging `cmd/backscroll/read.go`. +- Tests set HOME, BACKSCROLL_CONFIG_DIR, and BACKSCROLL_DATABASE_PATH to temporary locations and compare input bytes, mtimes, and sidecar inventories. +- Follow strict RED → GREEN → TRIANGULATE → REFACTOR. Run focused tests before `just check`, `just test`, and `just ci`. +- Commit commands below are future implementation instructions only. Do not stage or commit during planning. + +--- + +## File map + +| Path | Responsibility | +|---|---| +| `internal/models/indexed_record.go` | Canonical indexed record shared by storage queries and recovery without an import cycle. | +| `internal/storage/records.go` | Keep query ownership in storage while returning `models.IndexedRecord`. | +| `internal/compat/types.go` | Add only recovery types now consumed by this plan. | +| `internal/compat/recovery.go` | Compute identities and payload hashes and return a complete union plan or typed diagnostics. | +| `internal/compat/recovery_test.go` | Identity/equivalence/conflict/uninterpretable matrix and accounting. | +| `internal/storage/recovery_records.go` | Read each historical lineage into current canonical `models.IndexedRecord` without writing inputs. | +| `internal/storage/recovery_destination.go` | Create fresh destination, import one transaction, database-level verify, and independent reopen verify. | +| `internal/storage/recovery_destination_test.go` | Current-schema, count, FK, FTS, shape, queryability, and rollback tests. | +| `internal/recovery/recovery.go` | Resolve/dedupe paths, dry-run/apply orchestration, backup, atomic replacement, fsync, and report. | +| `internal/recovery/recovery_test.go` | No-write dry-run, same-path, immutable stranded source, backup, replacement, and failure invariants. | +| `cmd/backscroll/recover.go` | Cobra flags, config resolution, output formatting, and exit behavior. | +| `cmd/backscroll/recover_test.go` | Real CLI contract and report fields. | +| `cmd/backscroll/main.go` | Register `newRecoverCmd`. | +| `cmd/backscroll/index_policy.go` | Shared command classes, fresh inspection, stale refusal, and diagnostic rendering. | +| `cmd/backscroll/index_policy_test.go` | Complete indexed-consumer, output-mode, and cached-fallback matrix. | +| `cmd/backscroll/compat_diagnostics_test.go` | Read-only diagnostics, direct-read exemption, and executable continuation coverage. | +| `cmd/backscroll/{search,list,patterns,rebuild,purge,annotate,status,validate}.go` | Delegate index health and preparation to the shared policy. | +| `cmd/backscroll/sync_helpers.go` | Propagate every discovery/decode/sync failure instead of falling back to cached data. | +| `scripts/calibration-extract/main.go` | Inspect before direct indexed-record extraction. | +| `tests/fixtures/recovery/*.sql` | Active/stranded lineage and conflict fixtures copied into temporary databases by tests. | + +### Task 1: Introduce the recovery contract and shared canonical indexed record + +**Files:** +- Create: `internal/models/indexed_record.go` +- Modify: `internal/storage/records.go:9-107` +- Modify: `internal/storage/unit_test.go:525-627,1782-1814,2877-2908,3064-3066` +- Modify: `internal/compat/types.go` + +**Interfaces:** +- Consumes: Plan 1 `compat.SchemaShape`, `compat.Diagnostic`, and existing `storage.IndexedRecord` fields. +- Produces: `models.IndexedRecord`, `compat.RecoveryInput`, `compat.CanonicalRecord`, and `compat.RecoveryPlan`. `storage.IndexedRecordQuery` remains storage-owned; `func (d *Database) QueryIndexedRecords(q IndexedRecordQuery) ([]models.IndexedRecord, error)` is the only query signature change. + +- [ ] **Step 1: Write the compile-time ownership test first** + +Add `TestQueryIndexedRecordsReturnsCanonicalModel` in `internal/storage/unit_test.go`. Assign the result of `QueryIndexedRecords` to `var got []models.IndexedRecord` and assert all existing source/path/ordinal/role/text/project/UUID/timestamp/content-type sentinels survive. + +- [ ] **Step 2: Run RED** + +Run: `go test ./internal/storage -run '^TestQueryIndexedRecordsReturnsCanonicalModel$'` + +Expected: FAIL because `models.IndexedRecord` does not exist and `QueryIndexedRecords` returns the storage-local type. + +- [ ] **Step 3: Move only the canonical record and add recovery-owned types** + +```go +// internal/models/indexed_record.go +type IndexedRecord struct { + Source string + SourcePath string + Ordinal int64 + Role string + Text string + Project *string + UUID *string + Timestamp *string + ContentType string +} + +// internal/compat/types.go +const ( + CodeRecoveryConflict Code = "recovery_conflict" + CodeUninterpretableRow Code = "uninterpretable_row" +) + +type RecoveryInput struct { + Shape SchemaShape + Records []models.IndexedRecord + RowCount int +} + +type CanonicalRecord struct { + Record models.IndexedRecord + PayloadHash string +} + +type RecoveryPlan struct { + InputShapes []SchemaShape + Records []CanonicalRecord + ExactDuplicates int +} +``` + +Delete only the duplicate `storage.IndexedRecord` definition, import `internal/models` in `records.go`, and make scan locals plus the return type use `models.IndexedRecord`. Keep `IndexedRecordQuery` and SQL query construction in storage. Do not introduce manifest types or move unrelated storage models. + +- [ ] **Step 4: GREEN and triangulate existing query filters** + +Run: `go test ./internal/storage -run '^(TestQueryIndexedRecordsReturnsCanonicalModel|TestQueryIndexedRecords|TestQueryIndexedRecordsLimitAndOffset|TestQueryIndexedRecordsComplexFilters)$'` + +Expected: PASS; ownership changes without changing query behavior. + +- [ ] **Step 5: Run affected packages and commit the coherent ownership move** + +Run: `go test ./internal/models ./internal/storage ./internal/compat` + +Expected: PASS. + +```bash +git add internal/models/indexed_record.go internal/storage/records.go internal/storage/unit_test.go internal/compat/types.go +git commit -m "refactor(recovery): share canonical indexed record model" +``` + +### Task 2: Adapt every supported input row into the current canonical record shape + +**Files:** +- Create: `internal/storage/recovery_records.go` +- Create: `internal/storage/recovery_records_test.go` +- Create: `tests/fixtures/recovery/active-v13.sql` +- Create: `tests/fixtures/recovery/stranded-v3-no-source-metadata.sql` +- Create: `tests/fixtures/recovery/stranded-v7.sql` + +**Interfaces:** +- Consumes: Plan 1 `compat.InspectIndex`, Plan 1 `compat.SchemaShape`, and Task 1 `models.IndexedRecord`. +- Produces: Task 1 `compat.RecoveryInput` with fields `Shape compat.SchemaShape`, `Records []models.IndexedRecord`, and `RowCount int`; plus `func ReadRecoveryInput(ctx context.Context, db *Database) (compat.RecoveryInput, *compat.Diagnostic, error)`. + +- [ ] **Step 1: Write historical adaptation tests first** + +```go +func TestReadRecoveryInputAdaptsSupportedLineages(t *testing.T) { + for _, fixture := range []string{"active-v13.sql", "stranded-v3-no-source-metadata.sql", "stranded-v7.sql"} { + t.Run(fixture, func(t *testing.T) { + db := openRecoveryFixtureReadOnly(t, fixture) + got, diag, err := ReadRecoveryInput(context.Background(), db) + if err != nil || diag != nil { t.Fatalf("err=%v diagnostic=%+v", err, diag) } + if got.RowCount == 0 || got.RowCount != len(got.Records) { t.Fatalf("input=%+v", got) } + }) + } +} +``` + +Seed source, path, ordinal, role, text, project, UUID, timestamp, and content type sentinels and assert exact preservation. + +- [ ] **Step 2: Run RED** + +Run: `go test ./internal/storage -run '^TestReadRecoveryInput'` + +Expected: FAIL with `undefined: ReadRecoveryInput`. + +- [ ] **Step 3: Implement read-only adapters keyed by inspected shape** + +Use a switch over Plan 1’s stable lineage/signature identifier, not migration number alone. Each adapter runs SELECT-only queries and maps absent historical columns to the same canonical defaults current storage migration would produce. Unknown/readable shape returns `CodeUnsupportedLineage`; a row missing required canonical payload returns `CodeUninterpretableRow`. These internal diagnostics do not print command text or set `Continuation`; Task 8 renders them only after `recover` is functional. + +```go +func ReadRecoveryInput(ctx context.Context, db *Database) (compat.RecoveryInput, *compat.Diagnostic, error) { + plan, diag, err := compat.InspectIndex(ctx, db.DB()) + if err != nil || diag != nil { return compat.RecoveryInput{}, diag, err } + shape := plan.From + records, err := readRecordsForSignature(ctx, db.DB(), shape.Signature) + if err != nil { return compat.RecoveryInput{}, nil, err } + return compat.RecoveryInput{Shape: shape, Records: records, RowCount: len(records)}, nil, nil +} +``` + +- [ ] **Step 4: GREEN and triangulate unknown/corrupt rows** + +Run: `go test ./internal/storage -run '^(TestReadRecoveryInputAdaptsSupportedLineages|TestReadRecoveryInputRejectsUnknownShape|TestReadRecoveryInputRejectsMissingCanonicalPayload|TestReadRecoveryInputPerformsNoWrites)$'` + +Expected: PASS; SQLite `PRAGMA query_only` remains true and fixture bytes do not change. + +- [ ] **Step 5: Refactor adapter column lists and run storage tests** + +Run: `go test ./internal/storage` + +Expected: PASS. + +- [ ] **Step 6: Commit the canonical read adapters** + +```bash +git add internal/storage/recovery_records.go internal/storage/recovery_records_test.go tests/fixtures/recovery +git commit -m "feat(recovery): adapt supported indexes read-only" +``` + +### Task 3: Build the complete active-plus-stranded union plan + +**Files:** +- Create: `internal/compat/recovery.go` +- Create: `internal/compat/recovery_test.go` + +**Interfaces:** +- Consumes: `compat.RecoveryInput`, `compat.RecoveryPlan`, `compat.CanonicalRecord`, and `compat.Diagnostic` introduced by Task 1 and populated by Task 2. +- Produces: `func PlanRecovery(inputs []RecoveryInput) (RecoveryPlan, []Diagnostic, error)` and private exact identity type `type recordIdentity struct { Kind string; UUID string; SourcePath string; Ordinal int64 }`. This keeps `internal/compat` independent of `internal/storage` and avoids an import cycle. + +- [ ] **Step 1: Write the full identity/conflict matrix** + +Add `TestRecoverIdentityAndConflictMatrixAcrossInputs` with cases: valid UUID beats differing path/ordinal; empty UUID falls back to exact path/ordinal; equal identity/equal canonical payload collapses; equal identity/different payload conflicts; equal content under different identities remains two records; invalid non-empty UUID is uninterpretable; empty UUID plus empty path or negative ordinal is uninterpretable. Run each case both within one input and across active/stranded inputs. + +- [ ] **Step 2: Run RED** + +Run: `go test ./internal/compat -run '^TestRecoverIdentityAndConflictMatrixAcrossInputs$'` + +Expected: FAIL with `undefined: PlanRecovery`. + +- [ ] **Step 3: Implement identity and canonical payload hashing** + +Validate UUID using `uuid.Parse`; do not invent one. Serialize all canonical payload fields except identity in a fixed length-prefixed order and SHA-256 the bytes. Preserve exact stored `source_path`; do not normalize case, separators, symlinks, host, or project roots. + +```go +func identityOf(r models.IndexedRecord) (recordIdentity, error) { + if r.UUID != nil && *r.UUID != "" { + if _, err := uuid.Parse(*r.UUID); err != nil { return recordIdentity{}, err } + return recordIdentity{Kind: "uuid", UUID: *r.UUID}, nil + } + if r.SourcePath == "" || r.Ordinal < 0 { return recordIdentity{}, errUnsafeIdentity } + return recordIdentity{Kind: "path_ordinal", SourcePath: r.SourcePath, Ordinal: r.Ordinal}, nil +} +``` + +Sort output deterministically by identity kind, UUID, source path, ordinal, then payload hash. Set `InputShapes` active first and `ExactDuplicates` to the number collapsed. Return no applicable records when any diagnostic exists. + +- [ ] **Step 4: GREEN and triangulate complete accounting** + +Run: `go test ./internal/compat -run '^(TestRecoverIdentityAndConflictMatrixAcrossInputs|TestPlanRecoveryAccountsForEveryInputRow|TestPlanRecoveryPreservesDistinctHashEquivalentIdentities|TestPlanRecoveryIsDeterministic)$'` + +Expected: PASS; `sum(input rows) == len(plan.Records) + plan.ExactDuplicates` only when diagnostics are empty, while conflict/uninterpretable cases name every rejected identity. + +- [ ] **Step 5: Refactor hashing and run compat tests** + +Run: `go test ./internal/compat` + +Expected: PASS. + +- [ ] **Step 6: Commit the stateless union planner** + +```bash +git add internal/compat/recovery.go internal/compat/recovery_test.go +git commit -m "feat(recovery): plan deterministic active stranded union" +``` + +### Task 4: Add the exact Cobra command, path resolution, same-path deduplication, and dry-run + +**Files:** +- Create: `internal/recovery/recovery.go` +- Create: `internal/recovery/recovery_test.go` +- Create: `cmd/backscroll/recover.go` +- Create: `cmd/backscroll/recover_test.go` +- Modify: `cmd/backscroll/main.go:59-83` + +**Interfaces:** +- Consumes: `config.Config.DatabasePath`, `storage.OpenReadOnly`, `storage.ReadRecoveryInput`, `compat.PlanRecovery`. +- Produces: `type Options struct { ActivePath string; FromPath string; DryRun bool }`, `type Report struct { ActivePath string; BackupPath string; InputCounts []int; ExactDuplicates int; FinalCount int; Shapes []compat.SchemaShape; Conflicts []compat.Diagnostic }`, `func Execute(ctx context.Context, opts Options) (Report, error)`, and `func newRecoverCmd(stdout, stderr io.Writer) *cobra.Command`. + +- [ ] **Step 1: Write CLI and no-write dry-run tests** + +Add `TestRecoverDryRunMatchesUnionApplyPlanWithoutWrites` and `TestRecoverSameResolvedPathIsOneInput`. Before dry-run, inventory every file in the active directory with bytes and mtimes. Assert identical inventory afterward, `InputCounts` has one entry for same-path resolution, and stdout contains active path, per-input counts, duplicate count, final count, shapes, and intended replacement path. + +- [ ] **Step 2: Run RED** + +Run: `go test ./internal/recovery ./cmd/backscroll -run '^(TestRecoverDryRunMatchesUnionApplyPlanWithoutWrites|TestRecoverSameResolvedPathIsOneInput)$'` + +Expected: FAIL because packages/command do not exist and `buildRootCmd` has no `recover`. + +- [ ] **Step 3: Implement exact command and read-only planner path** + +`newRecoverCmd` uses `Use: "recover"`, `cobra.NoArgs`, required string flag `--from`, and bool `--dry-run`; it defines no other recovery flags. Resolve both paths with `filepath.Abs`, `filepath.EvalSymlinks` when possible, and `os.SameFile` after stat. Open distinct inputs via `storage.OpenReadOnly` only. Pass active first, stranded second; pass one input when paths resolve identically. + +```go +cmd.Flags().StringVar(&from, "from", "", "path to one stranded Backscroll database") +cmd.Flags().BoolVar(&dryRun, "dry-run", false, "plan and report recovery without writing files") +_ = cmd.MarkFlagRequired("from") +``` + +For dry-run, return immediately after planning/report construction and register `newRecoverCmd` in `buildRootCmd`. Do not activate shared index blocking or executable-continuation tests yet: apply becomes fully functional in Tasks 5–6, then Tasks 7–8 activate and prove the policy with no skip. + +- [ ] **Step 4: GREEN and triangulate forbidden command shapes** + +Run: `go test ./internal/recovery ./cmd/backscroll -run '^(TestRecoverDryRunMatchesUnionApplyPlanWithoutWrites|TestRecoverSameResolvedPathIsOneInput|TestRecoverRejectsMissingFrom|TestRecoverHasNoGeneralMergeFlags)$'` + +Expected: PASS; `--into`, repeated `--from`, `--force`, and `--partial` are rejected by Cobra. + +- [ ] **Step 5: Refactor report formatting and run command packages** + +Run: `go test ./internal/recovery ./cmd/backscroll` + +Expected: PASS. + +- [ ] **Step 6: Commit command and dry-run boundary** + +```bash +git add internal/recovery cmd/backscroll/recover.go cmd/backscroll/recover_test.go cmd/backscroll/main.go +git commit -m "feat(cli): add read-only stranded recovery dry run" +``` + +### Task 5: Import the verified union into one fresh current-schema transaction + +**Files:** +- Create: `internal/storage/recovery_destination.go` +- Create: `internal/storage/recovery_destination_test.go` +- Modify: `internal/recovery/recovery.go` + +**Interfaces:** +- Consumes: applicable `compat.RecoveryPlan` and existing current-schema `storage.Open`/`SyncFiles` primitives. +- Produces: `func CreateRecoveryDestination(ctx context.Context, dir string, plan compat.RecoveryPlan) (path string, err error)` and `func VerifyRecoveryDestination(ctx context.Context, path string, plan compat.RecoveryPlan) error`. + +- [ ] **Step 1: Write fresh-destination and rollback tests** + +Add `TestRecoverUnionPreservesActiveAndStrandedRecords` and `TestRecoverConflictOrUninterpretableRollsBackEverything`. Seed unique active/stranded records plus one exact duplicate; assert fresh destination contains both unique sets once. Inject an import failure and assert no committed destination and unchanged inputs. + +- [ ] **Step 2: Run RED** + +Run: `go test ./internal/storage ./internal/recovery -run '^(TestRecoverUnionPreservesActiveAndStrandedRecords|TestRecoverConflictOrUninterpretableRollsBackEverything)$'` + +Expected: FAIL with `undefined: CreateRecoveryDestination`. + +- [ ] **Step 3: Implement one-transaction destination creation and verification** + +Create the temp file with `os.CreateTemp(dir, ".backscroll-recover-*.db")`, close it, initialize current schema, begin one SQL transaction, insert the canonical union through transaction-aware storage helpers, verify source accounting, union row count, unique identities, `PRAGMA foreign_key_check`, FTS row/count consistency, current schema signature, and representative exact queryability, then commit once. + +Close the destination and independently reopen with `OpenReadOnly`; `VerifyRecoveryDestination` reruns row count, identity, FTS, shape, and representative query checks against committed bytes. Return the path only after independent verification. + +- [ ] **Step 4: GREEN and triangulate post-commit independent failure** + +Run: `go test ./internal/storage ./internal/recovery -run '^(TestRecoverUnionPreservesActiveAndStrandedRecords|TestRecoverConflictOrUninterpretableRollsBackEverything|TestRecoveryDestinationStartsFreshAtCurrentSchema|TestRecoveryDestinationIndependentVerificationRejectsTamper)$'` + +Expected: PASS; tampering between close and independent verification prevents replacement. + +- [ ] **Step 5: Refactor transaction helpers and run affected packages** + +Run: `go test ./internal/storage ./internal/recovery` + +Expected: PASS. + +- [ ] **Step 6: Commit fresh destination apply logic** + +```bash +git add internal/storage/recovery_destination.go internal/storage/recovery_destination_test.go internal/recovery/recovery.go +git commit -m "feat(recovery): build verified union destination" +``` + +### Task 6: Preserve active backup and atomically replace on the same filesystem + +**Files:** +- Modify: `internal/recovery/recovery.go` +- Modify: `internal/recovery/recovery_test.go` +- Modify: `cmd/backscroll/recover_test.go` + +**Interfaces:** +- Consumes: independently verified sibling destination from Task 5. +- Produces: `func replaceActiveWithBackup(activePath, verifiedTempPath string) (backupPath string, err error)` and complete success report. + +- [ ] **Step 1: Write replacement and immutability tests** + +Add `TestRecoverAtomicallyReplacesAndPreservesActiveBackup` and `TestRecoverStrandedSourceIsReadOnly`. Capture active and stranded bytes, mtimes, inode/file IDs where available, and `-wal`/`-shm` sidecar inventories across dry-run, success, conflict failure, and injected replacement failure. Assert backup bytes equal original active bytes and stranded metadata never changes. + +- [ ] **Step 2: Run RED** + +Run: `go test ./internal/recovery ./cmd/backscroll -run '^(TestRecoverAtomicallyReplacesAndPreservesActiveBackup|TestRecoverStrandedSourceIsReadOnly)$'` + +Expected: FAIL because apply does not yet back up or replace active. + +- [ ] **Step 3: Implement backup, fsync, and replacement** + +Close every input and destination handle first. Choose a unique sibling backup name `.backscroll.db.backup--` without overwriting. Rename active to backup, fsync the directory, rename verified temp to active, fsync again. If the second rename fails, rename backup back to active and fsync; return an explicit error whether restoration succeeds or leaves a restorable backup. + +```go +func syncDir(path string) error { + f, err := os.Open(path) + if err != nil { return err } + defer f.Close() + return f.Sync() +} +``` + +Assert `filepath.Dir(activePath) == filepath.Dir(verifiedTempPath)` before either rename; reject cross-filesystem paths rather than copying. + +- [ ] **Step 4: GREEN and triangulate replacement failure** + +Run: `go test ./internal/recovery ./cmd/backscroll -run '^(TestRecoverAtomicallyReplacesAndPreservesActiveBackup|TestRecoverStrandedSourceIsReadOnly|TestRecoverReplacementFailureRestoresActive|TestRecoverNeverDeletesBackup)$'` + +Expected: PASS; success reports active/backup paths and counts, while failure retains a valid active path or names the exact restorable backup. + +- [ ] **Step 5: Refactor platform-specific rename injection and run packages** + +Run: `go test ./internal/recovery ./cmd/backscroll` + +Expected: PASS. + +- [ ] **Step 6: Commit atomic replacement as one rollback boundary** + +```bash +git add internal/recovery/recovery.go internal/recovery/recovery_test.go cmd/backscroll/recover_test.go +git commit -m "feat(recovery): atomically replace active with preserved backup" +``` + +### Task 7: Activate one fail-closed policy across every indexed consumer + +**Files:** +- Create: `cmd/backscroll/index_policy.go` +- Create: `cmd/backscroll/index_policy_test.go` +- Modify: `cmd/backscroll/search.go:99-145` +- Modify: `cmd/backscroll/list.go:62-88` +- Modify: `cmd/backscroll/patterns.go:87-137` +- Modify: `cmd/backscroll/rebuild.go:31-109` +- Modify: `cmd/backscroll/purge.go:35-50` +- Modify: `cmd/backscroll/annotate.go:48-69` +- Modify: `cmd/backscroll/sync_helpers.go:16-160` +- Modify: `scripts/calibration-extract/main.go:39-49` + +**Interfaces:** +- Consumes: Plan 1 `storage.OpenCompatible`, Task 6’s functional `recover`, existing `maybeAutoSync`, and typed `compat.Diagnostic` values. +- Produces: `type indexCommandClass uint8`, `func prepareIndex(ctx context.Context, cfg *config.Config, class indexCommandClass, autoSync bool) (*storage.Database, *compat.Diagnostic, error)`, `func continuationFor(d compat.Diagnostic, activePath string) compat.Diagnostic`, and `func writeDiagnostic(stdout, stderr io.Writer, d compat.Diagnostic, jsonMode bool) error`. + +- [ ] **Step 1: Add the complete stale-consumer failure matrix** + +```go +func TestStaleIndexBlocksIndexBackedCommands(t *testing.T) { + cases := []struct{ name string; argv []string; mutation bool }{ + {"search", []string{"search", "sentinel"}, false}, + {"search-indexed-only", []string{"search", "sentinel", "--indexed-only"}, false}, + {"search-json", []string{"search", "sentinel", "--json"}, false}, + {"search-robot", []string{"search", "sentinel", "--robot"}, false}, + {"list", []string{"list"}, false}, + {"list-json", []string{"list", "--json"}, false}, + {"patterns", []string{"patterns", "--kind", "commands"}, false}, + {"rebuild", []string{"rebuild"}, true}, + {"purge", []string{"purge", "--before", "2030-01-01"}, true}, + {"annotate", []string{"annotate", "--uuid", "u", "--kind", "correction", "--label", "x"}, true}, + } + // Seed cached sentinel output, force migration or sync failure, execute argv, + // assert non-zero, no sentinel output, and byte-identical DB for mutations. +} +``` + +- [ ] **Step 2: Run RED and observe the cached fallback** + +Run: `go test ./cmd/backscroll -run '^TestStaleIndexBlocksIndexBackedCommands$'` + +Expected: FAIL because indexed reads warn and continue with cached rows and mutations open the database outside one shared policy. + +- [ ] **Step 3: Implement fresh preparation and remove every cached fallback** + +Define `indexDataRead`, `indexMutation`, `indexDiagnostic`, and `indexRemediation`. `prepareIndex` performs fresh inspection on every invocation, applies Plan 1 migration only for data/mutation preparation, and returns before any query, mutation, success output, or zero-result claim on inspection, migration, discovery, decode, or sync failure. It never caches diagnostics. + +`continuationFor` receives the resolved active path and sets exact argv `[]string{"recover", "--from", activePath, "--dry-run"}` only now that `recover` exists. It rejects an empty path and never writes a literal placeholder. Replace direct opens in the listed indexed commands. Make `maybeAutoSync` return unknown-reader, discovery, hash, parse, and `SyncFiles` errors instead of logging and continuing. Plan 2 later adds all-manifest preflight but consumes this already-active fail-closed boundary. The calibration extractor calls `compat.InspectIndex` before `QueryIndexedRecords` and aborts on every diagnostic. + +- [ ] **Step 4: GREEN and triangulate output-mode bypasses** + +Run: `go test ./cmd/backscroll ./scripts/calibration-extract -run '^(TestStaleIndexBlocksIndexBackedCommands|TestIndexedOnlyDoesNotBypassStaleBlock|TestMachineModesCarryDiagnosticCodeAndContinuation|TestCalibrationExtractRejectsUnsupportedIndex)$'` + +Expected: PASS; indexed-only, JSON, robot, source filters, and empty-result paths cannot bypass refusal or emit cached data. + +- [ ] **Step 5: Refactor adapters and run affected packages** + +Run: `go test ./cmd/backscroll ./scripts/calibration-extract` + +Expected: PASS with no skipped tests. + +- [ ] **Step 6: Commit the activation as one rollback unit** + +```bash +git add cmd/backscroll/index_policy.go cmd/backscroll/index_policy_test.go cmd/backscroll/search.go cmd/backscroll/list.go cmd/backscroll/patterns.go cmd/backscroll/rebuild.go cmd/backscroll/purge.go cmd/backscroll/annotate.go cmd/backscroll/sync_helpers.go scripts/calibration-extract/main.go +git commit -m "fix(cli): block every stale index consumer" +``` + +### Task 8: Make diagnostics read-only and prove every continuation and direct-read exemption + +**Files:** +- Modify: `cmd/backscroll/status.go:44-147` +- Modify: `cmd/backscroll/validate.go:37-69` +- Modify: `cmd/backscroll/main_test.go:20-95` +- Create: `cmd/backscroll/compat_diagnostics_test.go` +- Test unchanged: `cmd/backscroll/read.go:12-113` + +**Interfaces:** +- Consumes: `prepareIndex(..., indexDiagnostic, false)`, `continuationFor`, `writeDiagnostic`, and the registered `newRecoverCmd`. +- Produces: read-only unhealthy `status`/`validate` behavior plus `TestDirectReadRemainsAvailableButClaimsNoIndexFreshness` and `TestBlockingDiagnosticsHaveExecutableContinuations`. + +- [ ] **Step 1: Write the diagnostic and direct-read tests** + +`TestDirectReadRemainsAvailableButClaimsNoIndexFreshness` creates an unsupported database plus a direct JSONL fixture, executes the existing `read`, and asserts decoded file content appears while `usable`, `fresh`, and `current index` claims do not. It also asserts the unsupported database bytes remain unchanged. + +`TestBlockingDiagnosticsHaveExecutableContinuations` builds each real unhealthy scenario rather than enumerating hypothetical codes: unsupported lineage, migration failure, stale sync, recovery conflict, and uninterpretable row. For each emitted diagnostic it requires non-empty argv, builds a fresh `buildRootCmd`, sets the exact argv, executes Cobra, and accepts a domain remediation error only after command/flag resolution. Every subcase must execute and pass at this task boundary. + +- [ ] **Step 2: Run RED** + +Run: `go test ./cmd/backscroll -run '^(TestDirectReadRemainsAvailableButClaimsNoIndexFreshness|TestBlockingDiagnosticsHaveExecutableContinuations)$'` + +Expected: FAIL because `status`/`validate` still mutate or continue and diagnostic continuation rendering is not yet centralized. The failure must not be an unresolved `recover` command because Task 4 already registered it. + +- [ ] **Step 3: Make diagnostic commands read-only without touching direct read** + +Remove auto-sync and write-mode `storage.Open` from `status` and `validate`. Both inspect read-only, print the primary typed diagnostic plus independently safe secondary diagnostics, emit the same `code` and `continuation` fields in JSON, and exit non-zero when unhealthy. Keep `cmd/backscroll/read.go` byte-for-byte unchanged: its existing `newReadCmd`, `runRead`, and `runReadSemantic` continue to avoid `prepareIndex`. + +Update only the shared `testEnv` helper in `main_test.go` to use `t.Setenv` and always set HOME, BACKSCROLL_CONFIG_DIR, and BACKSCROLL_DATABASE_PATH under temporary directories. + +- [ ] **Step 4: GREEN and triangulate no-write diagnostics** + +Run: `go test ./cmd/backscroll -run '^(TestDirectReadRemainsAvailableButClaimsNoIndexFreshness|TestStatusUnhealthyIsReadOnly|TestValidateUnhealthyIsReadOnly|TestBlockingDiagnosticsHaveExecutableContinuations)$'` + +Expected: PASS with no skipped tests; no status/validate invocation changes database bytes or sidecars, and every printed continuation resolves through Cobra. + +- [ ] **Step 5: Run the CLI package** + +Run: `go test ./cmd/backscroll` + +Expected: PASS with no real user config access. + +- [ ] **Step 6: Commit diagnostics and proof without staging `read.go`** + +```bash +git add cmd/backscroll/status.go cmd/backscroll/validate.go cmd/backscroll/main_test.go cmd/backscroll/compat_diagnostics_test.go +git commit -m "fix(cli): expose safe recovery from incompatible indexes" +``` + +### Task 9: Close issue #32 and prove the activated compatibility boundary + +**Files:** +- Modify: `internal/compat/recovery_test.go` +- Modify: `internal/storage/recovery_destination_test.go` +- Modify: `internal/recovery/recovery_test.go` +- Modify: `cmd/backscroll/recover_test.go` +- Modify: `cmd/backscroll/index_policy_test.go` +- Modify: `cmd/backscroll/compat_diagnostics_test.go` + +**Interfaces:** +- Consumes: all Plan 3 behavior plus Plan 1’s migration evidence. +- Produces: complete named #32 closure evidence, active migration/blocking/direct-read/continuation evidence for the migration-and-operability portion of #31, and final command contracts for Plans 2 and 4. + +- [ ] **Step 1: Run all named #32 tests together** + +Run: `go test ./internal/compat ./internal/storage ./internal/recovery ./cmd/backscroll -run '^(TestRecoverDryRunMatchesUnionApplyPlanWithoutWrites|TestRecoverUnionPreservesActiveAndStrandedRecords|TestRecoverIdentityAndConflictMatrixAcrossInputs|TestRecoverConflictOrUninterpretableRollsBackEverything|TestRecoverSameResolvedPathIsOneInput|TestRecoverAtomicallyReplacesAndPreservesActiveBackup|TestRecoverStrandedSourceIsReadOnly)$'` + +Expected: PASS with no skipped case. + +- [ ] **Step 2: Run the activated migration and command-policy boundary** + +Run: `go test ./internal/compat ./internal/storage ./cmd/backscroll -run '^(TestCheckedInReleaseSchemaManifestIsComplete|TestPublishedGoLineagesUpgradeLosslessly|TestHistoricalLineageWithoutSourceMetadataUpgradesLosslessly|TestMigrationSnapshotAndRollbackOnDestructiveFailure|TestStaleIndexBlocksIndexBackedCommands|TestDirectReadRemainsAvailableButClaimsNoIndexFreshness|TestBlockingDiagnosticsHaveExecutableContinuations)$'` + +Expected: PASS with no skipped test and no cached fallback. This proves the migration and operability portions of #31 but does not close #31; Plan 2 must still supply manifest-only ingestion evidence. + +- [ ] **Step 3: Run repository gates** + +Run: `just check` + +Expected: PASS. + +Run: `just test` + +Expected: PASS. + +Run: `just ci` + +Expected: PASS. + +- [ ] **Step 4: Record exact issue closure evidence** + +The implementation PR lists the seven named recovery tests, confirms active-only and stranded-only preservation, exact-duplicate collapse, same-path single accounting, UUID/path-ordinal identity, hash-equivalence-only behavior, conflict/uninterpretable all-or-nothing abort, dry-run no writes, immutable stranded source, fresh V13 destination, independent verification, active backup, and same-filesystem atomic replacement. It also lists the migration/blocking/direct-read/executable-continuation tests, states every test passed without skips, and says: “Issue #31 remains open until Plan 2’s ingestion-half tests pass.” + +- [ ] **Step 5: Commit only if closure assertions changed** + +```bash +git add internal/compat/recovery_test.go internal/storage/recovery_destination_test.go internal/recovery/recovery_test.go cmd/backscroll/recover_test.go cmd/backscroll/index_policy_test.go cmd/backscroll/compat_diagnostics_test.go +git commit -m "test(recovery): prove active stranded atomic union" +``` diff --git a/docs/superpowers/plans/2026-08-19-recovered-source-accounting.md b/docs/superpowers/plans/2026-08-19-recovered-source-accounting.md new file mode 100644 index 0000000..bc45ce3 --- /dev/null +++ b/docs/superpowers/plans/2026-08-19-recovered-source-accounting.md @@ -0,0 +1,719 @@ +# Recovered Source Accounting Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make every successfully recovered database pass public validation while preserving backfill eligibility and forcing any still-present source to be genuinely re-synced. + +**Architecture:** Recovery writes one provisional `indexed_files` row per distinct recovered `source_path`, using the reserved non-SHA marker `backscroll:recovered` and a NULL `last_indexed`. Recovery verification requires an exact marker set; hash- and status-oriented consumers exclude markers, backfill includes them, and the existing `SyncFiles` upsert replaces a marker with a real hash. + +**Tech Stack:** Go, cobra command tests, `database/sql`, modernc.org/sqlite, SQLite FTS5, stdlib `testing`. + +**Spec:** `docs/superpowers/specs/2026-08-19-recovered-source-accounting-design.md` + +## Global Constraints + +- Recovery reconstructs database state only; it must never create, restore, read, or modify original JSONL or Markdown source files. +- The provisional hash value is exactly `backscroll:recovered` and must never look like a 64-character hexadecimal SHA-256. +- Provisional rows use `NULL` for `last_indexed`. +- Public validation must continue rejecting genuine `search_items` rows with no matching accounting path. +- Recovered paths remain eligible for `BackfillDerived()`. +- `GetFileHashes()` and `GetStats()` must not treat provisional rows as sources indexed from disk. +- A normal `SyncFiles()` call must replace provisional accounting with the real hash in the same transaction as the synced records. +- Do not add a migration or change the SQLite schema. +- Preserve deterministic recovery output, canonical records, FTS queryability, atomic installation, and byte-identical active backup behavior. + +## File Structure + +- Create `internal/storage/recovery_accounting.go`: owns the reserved marker and the predicate that identifies provisional source accounting. +- Modify `internal/storage/recovery_destination.go`: writes deterministic provisional accounting and verifies its exact committed shape. +- Modify `internal/storage/recovery_destination_test.go`: pins marker creation, NULL timestamp, exact-path verification, and tamper rejection. +- Modify `internal/storage/sync.go`: excludes provisional markers from the autosync hash skip map; normal sync upsert remains the transition mechanism. +- Modify `internal/storage/queries.go`: excludes provisional markers from file-count and last-indexed statistics. +- Modify `internal/storage/backfill.go`: treats provisional markers like missing/expired source accounting. +- Modify `internal/storage/storage_test.go`: tests hash-map filtering, status filtering, and replacement by normal sync. +- Modify `internal/storage/backfill_test.go`: proves marked recovered paths remain eligible for derived-data backfill. +- Modify `cmd/backscroll/recover_test.go`: adds the end-to-end v13 + v7 recovery-to-validation regression and verifies backup/rows/FTS. + +--- + +### Task 1: Write and verify provisional recovery accounting + +**Files:** +- Create: `internal/storage/recovery_accounting.go` +- Modify: `internal/storage/recovery_destination.go:116-136, 230-305` +- Test: `internal/storage/recovery_destination_test.go:330-460, 660-725` + +**Interfaces:** +- Consumes: `compat.RecoveryPlan`, `compat.CanonicalRecord.Record.SourcePath`, the existing recovery transaction, and `compat.Queryer`. +- Produces: `const recoveredSourceHash = "backscroll:recovered"`, `func isRecoveredSourceHash(string) bool`, `func recoveryDestinationSourcePaths(compat.RecoveryPlan) []string`, `func insertRecoveryDestinationAccounting(context.Context, *sql.Tx, compat.RecoveryPlan) error`, and exact accounting checks inside `verifyRecoveryDestinationRows`. + +- [ ] **Step 1: Add failing destination-creation assertions** + +Update `TestRecoveryDestinationStartsFreshAtCurrentSchema` and `assertRecoveryDestinationRecords` so they expect one marker per distinct planned path instead of zero `indexed_files` rows. Use a query that also verifies `last_indexed` is NULL: + +```go +rows, err := db.DB().Query(` + SELECT path, hash, last_indexed + FROM indexed_files + ORDER BY path +`) +if err != nil { + t.Fatalf("query recovered source accounting: %v", err) +} +defer func() { _ = rows.Close() }() + +var gotPaths []string +for rows.Next() { + var path, hash string + var lastIndexed sql.NullString + if err := rows.Scan(&path, &hash, &lastIndexed); err != nil { + t.Fatalf("scan recovered source accounting: %v", err) + } + if hash != recoveredSourceHash || lastIndexed.Valid { + t.Fatalf("accounting for %s = hash %q last_indexed %+v", path, hash, lastIndexed) + } + gotPaths = append(gotPaths, path) +} +``` + +Build `wantPaths` from the plan's distinct `SourcePath` values, sort it, and compare with `reflect.DeepEqual(gotPaths, wantPaths)`. Add `database/sql` to the test imports if it is not already present. + +- [ ] **Step 2: Add failing tamper cases for missing and incorrect accounting** + +Add a table-driven test next to `TestRecoveryDestinationIndependentVerificationRejectsTamper`: + +```go +func TestRecoveryDestinationVerificationRejectsInvalidRecoveredAccounting(t *testing.T) { + cases := []struct { + name string + mutate string + }{ + { + name: "missing path", + mutate: `DELETE FROM indexed_files WHERE path = '/sessions/source.jsonl';`, + }, + { + name: "real-looking but unverified hash", + mutate: `UPDATE indexed_files SET hash = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' WHERE path = '/sessions/source.jsonl';`, + }, + { + name: "non-null last indexed", + mutate: `UPDATE indexed_files SET last_indexed = '2026-08-19T00:00:00Z' WHERE path = '/sessions/source.jsonl';`, + }, + } + // For every case, create a one-path recovery destination, mutate it, + // and require VerifyRecoveryDestination(ctx, destPath, plan) to fail. +} +``` + +Keep the existing invented-extra-path tamper test; it covers the fourth exact-set failure mode. + +- [ ] **Step 3: Run the focused tests and confirm RED** + +Run: + +```bash +go test ./internal/storage -run 'TestRecoveryDestination(Start|Verification|Independent)' -count=1 +``` + +Expected: FAIL because recovery still produces zero accounting rows and the new marker identifiers do not exist. + +- [ ] **Step 4: Create the marker contract** + +Create `internal/storage/recovery_accounting.go`: + +```go +package storage + +const recoveredSourceHash = "backscroll:recovered" + +func isRecoveredSourceHash(hash string) bool { + return hash == recoveredSourceHash +} +``` + +Keep the marker unexported because it is an internal storage representation, not CLI or package API. + +- [ ] **Step 5: Insert deterministic provisional accounting** + +In `internal/storage/recovery_destination.go`, add these helpers: + +```go +func recoveryDestinationSourcePaths(plan compat.RecoveryPlan) []string { + seen := make(map[string]struct{}, len(plan.Records)) + for _, planned := range plan.Records { + seen[planned.Record.SourcePath] = struct{}{} + } + paths := make([]string, 0, len(seen)) + for path := range seen { + paths = append(paths, path) + } + sort.Strings(paths) + return paths +} + +func insertRecoveryDestinationAccounting(ctx context.Context, tx *sql.Tx, plan compat.RecoveryPlan) error { + for _, path := range recoveryDestinationSourcePaths(plan) { + if _, err := tx.ExecContext(ctx, ` + INSERT INTO indexed_files (path, hash, last_indexed) + VALUES (?, ?, NULL) + `, path, recoveredSourceHash); err != nil { + return fmt.Errorf("insert recovered source accounting for %s: %w", path, err) + } + } + return nil +} +``` + +Call `insertRecoveryDestinationAccounting(ctx, tx, plan)` immediately after the canonical record insertion loop and before `verifyRecoveryDestinationQueryer`. Wrap failure as `insert recovery source accounting: %w`. + +- [ ] **Step 6: Replace the zero-row verifier with exact marker verification** + +In `verifyRecoveryDestinationRows`, remove the `COUNT(*) == 0` block. Query all accounting rows ordered by path, scan `last_indexed` as `sql.NullString`, and compare against `recoveryDestinationSourcePaths(plan)`: + +```go +rows, err := q.QueryContext(ctx, ` + SELECT path, hash, last_indexed + FROM indexed_files + ORDER BY path +`) +if err != nil { + return fmt.Errorf("query recovered source accounting: %w", err) +} + +var gotPaths []string +for rows.Next() { + var path, hash string + var lastIndexed sql.NullString + if err := rows.Scan(&path, &hash, &lastIndexed); err != nil { + _ = rows.Close() + return fmt.Errorf("scan recovered source accounting: %w", err) + } + if !isRecoveredSourceHash(hash) { + _ = rows.Close() + return fmt.Errorf("recovered source accounting for %s has hash %q", path, hash) + } + if lastIndexed.Valid { + _ = rows.Close() + return fmt.Errorf("recovered source accounting for %s has last_indexed %q", path, lastIndexed.String) + } + gotPaths = append(gotPaths, path) +} +if err := rows.Err(); err != nil { + _ = rows.Close() + return fmt.Errorf("read recovered source accounting: %w", err) +} +if err := rows.Close(); err != nil { + return fmt.Errorf("close recovered source accounting: %w", err) +} +wantPaths := recoveryDestinationSourcePaths(plan) +if !reflect.DeepEqual(gotPaths, wantPaths) { + return fmt.Errorf("recovered source accounting paths do not match plan") +} +``` + +Do not add `reflect` solely for this comparison if a small existing slice-equality helper is clearer; if using `reflect.DeepEqual`, add the import explicitly. Ensure rows are closed before later queries on the single SQLite connection. + +- [ ] **Step 7: Run destination tests and confirm GREEN** + +Run: + +```bash +go test ./internal/storage -run 'TestRecoveryDestination|TestPublishedGoLineagesUpgradeLosslessly' -count=1 +``` + +Expected: PASS, including the existing canonical-row, FTS, cleanup, and tamper tests. + +- [ ] **Step 8: Commit Task 1** + +```bash +git add internal/storage/recovery_accounting.go internal/storage/recovery_destination.go internal/storage/recovery_destination_test.go +git commit -m "fix(storage): account for recovered source paths" +``` + +--- + +### Task 2: Make sync, status, and backfill marker-aware + +**Files:** +- Modify: `internal/storage/sync.go:320-342` +- Modify: `internal/storage/queries.go:25-48` +- Modify: `internal/storage/backfill.go:50-65` +- Test: `internal/storage/storage_test.go:350-425` +- Test: `internal/storage/backfill_test.go:1-50, 250-295` + +**Interfaces:** +- Consumes: `isRecoveredSourceHash(string) bool` and `recoveredSourceHash` from Task 1. +- Produces: `GetFileHashes()` containing only real source hashes, `GetStats()` containing only genuinely indexed file accounting, and `BackfillDerived()` eligibility for missing or provisionally recovered paths. `SyncFiles([]IndexedFile)` remains unchanged and replaces markers through its existing upsert. + +- [ ] **Step 1: Add failing hash-map and stats tests** + +Extend `TestGetFileHashes` after the normal sync: + +```go +if _, err := db.db.Exec(` + INSERT INTO indexed_files (path, hash, last_indexed) + VALUES ('/path/to/recovered.jsonl', ?, NULL) +`, recoveredSourceHash); err != nil { + t.Fatalf("insert recovered accounting: %v", err) +} + +hashes, err := db.GetFileHashes() +if err != nil { + t.Fatalf("failed to get hashes: %v", err) +} +if _, ok := hashes["/path/to/recovered.jsonl"]; ok { + t.Fatalf("GetFileHashes returned provisional recovered path") +} +``` + +Add `TestGetStatsExcludesRecoveredSourceAccounting`: + +```go +func TestGetStatsExcludesRecoveredSourceAccounting(t *testing.T) { + db, cleanup := newTestDB(t) + defer cleanup() + + if _, err := db.db.Exec(` + INSERT INTO indexed_files (path, hash, last_indexed) VALUES + ('/real.jsonl', 'real-hash', '2026-08-18T12:00:00Z'), + ('/recovered.jsonl', ?, NULL) + `, recoveredSourceHash); err != nil { + t.Fatal(err) + } + stats, err := db.GetStats() + if err != nil { + t.Fatal(err) + } + if stats.TotalFiles != 1 { + t.Fatalf("TotalFiles = %d, want 1", stats.TotalFiles) + } + want := time.Date(2026, 8, 18, 12, 0, 0, 0, time.UTC) + if !stats.IndexedAt.Equal(want) { + t.Fatalf("IndexedAt = %s, want %s", stats.IndexedAt, want) + } +} +``` + +- [ ] **Step 2: Add a normal-sync replacement regression** + +Add `database/sql` to `storage_test.go` imports, then add `TestSyncFilesReplacesRecoveredSourceAccounting`: + +```go +func TestSyncFilesReplacesRecoveredSourceAccounting(t *testing.T) { + db, cleanup := newTestDB(t) + defer cleanup() + + const path = "/path/to/recovered.jsonl" + if _, err := db.db.Exec(` + INSERT INTO indexed_files (path, hash, last_indexed) + VALUES (?, ?, NULL) + `, path, recoveredSourceHash); err != nil { + t.Fatal(err) + } + err := db.SyncFiles([]IndexedFile{{ + SourcePath: path, + Source: "session", + Hash: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + Project: "proj", + Messages: []IndexedMessage{{ + Ordinal: 0, Role: "user", Text: "resynced", UUID: getTestUUID(), ContentType: "text", + }}, + }}) + if err != nil { + t.Fatal(err) + } + var hash string + var lastIndexed sql.NullString + if err := db.db.QueryRow(`SELECT hash, last_indexed FROM indexed_files WHERE path = ?`, path).Scan(&hash, &lastIndexed); err != nil { + t.Fatal(err) + } + if hash == recoveredSourceHash || !lastIndexed.Valid { + t.Fatalf("accounting after sync = hash %q last_indexed %+v", hash, lastIndexed) + } +} +``` + +This test should already pass once the marker exists, proving that no special `SyncFiles` branch is necessary. + +Also add `TestPurgeRemovesRecoveredSourceAccounting` to pin the final lifecycle transition: + +```go +func TestPurgeRemovesRecoveredSourceAccounting(t *testing.T) { + db, cleanup := newTestDB(t) + defer cleanup() + + const path = "/path/to/purged-recovery.jsonl" + if _, err := db.db.Exec(` + INSERT INTO search_items + (source, source_path, ordinal, role, text, timestamp, uuid, project, content_type) + VALUES ('session', ?, 0, 'user', 'old recovered row', '2026-01-01T00:00:00Z', + '33333333-3333-4333-8333-333333333333', 'proj', 'text') + `, path); err != nil { + t.Fatal(err) + } + if _, err := db.db.Exec(` + INSERT INTO indexed_files (path, hash, last_indexed) + VALUES (?, ?, NULL) + `, path, recoveredSourceHash); err != nil { + t.Fatal(err) + } + if _, err := db.Purge("2026-01-02T00:00:00Z"); err != nil { + t.Fatal(err) + } + var count int + if err := db.db.QueryRow(`SELECT COUNT(*) FROM indexed_files WHERE path = ?`, path).Scan(&count); err != nil { + t.Fatal(err) + } + if count != 0 { + t.Fatalf("recovered accounting survived purge") + } +} +``` + +- [ ] **Step 3: Add a failing marked-path backfill test** + +Add `TestBackfillDerivedMinesRecoveredMarkedPath` beside `TestBackfillDerivedMinesTemplatesFromExpiredFile`: + +```go +func TestBackfillDerivedMinesRecoveredMarkedPath(t *testing.T) { + db, err := Open(filepath.Join(t.TempDir(), "test.db")) + if err != nil { + t.Fatal(err) + } + defer func() { _ = db.Close() }() + + if _, err := db.db.Exec(` + INSERT INTO search_items + (source, source_path, ordinal, role, text, timestamp, uuid, project, content_type, extraction_version) + VALUES ('session', '/recovered/s.jsonl', 0, 'assistant', 'error: recovered failure 42', + '2026-01-01T00:00:00Z', 'recovered#t0', 'proj', 'tool', 1) + `); err != nil { + t.Fatal(err) + } + if _, err := db.db.Exec(` + INSERT INTO indexed_files (path, hash, last_indexed) + VALUES ('/recovered/s.jsonl', ?, NULL) + `, recoveredSourceHash); err != nil { + t.Fatal(err) + } + + if err := db.BackfillDerived(BackfillDerivedOpts{}); err != nil { + t.Fatal(err) + } + var count int + if err := db.db.QueryRow(`SELECT COUNT(*) FROM message_templates`).Scan(&count); err != nil { + t.Fatal(err) + } + if count == 0 { + t.Fatal("marked recovered path was excluded from backfill") + } +} +``` + +- [ ] **Step 4: Run the focused tests and confirm RED where behavior is missing** + +Run: + +```bash +go test ./internal/storage -run 'Test(GetFileHashes|GetStatsExcludesRecovered|SyncFilesReplacesRecovered|PurgeRemovesRecovered|BackfillDerivedMinesRecovered)' -count=1 +``` + +Expected: hash-map, stats, and backfill tests FAIL; sync replacement PASS demonstrates the existing transition. + +- [ ] **Step 5: Filter provisional rows from the autosync skip map** + +Change `GetFileHashes` in `internal/storage/sync.go`: + +```go +rows, err := d.db.Query(` + SELECT path, hash + FROM indexed_files + WHERE hash <> ? +`, recoveredSourceHash) +``` + +Keep its return type and scan/error behavior unchanged. + +- [ ] **Step 6: Filter provisional rows from status statistics** + +Change both `GetStats` queries in `internal/storage/queries.go`: + +```go +err := d.db.QueryRow(` + SELECT COUNT(*) + FROM indexed_files + WHERE hash <> ? +`, recoveredSourceHash).Scan(&stats.TotalFiles) +``` + +```go +err = d.db.QueryRow(` + SELECT MAX(last_indexed) + FROM indexed_files + WHERE hash <> ? +`, recoveredSourceHash).Scan(&lastIndexed) +``` + +Do not alter message, chunk, embedding, or vector counts. + +- [ ] **Step 7: Include provisional paths in expired/recovery backfill discovery** + +Change the `BackfillDerived` discovery predicate and pass the marker as a parameter: + +```go +rows, err := d.db.Query(` + SELECT DISTINCT si.source_path, si.source + FROM search_items si + LEFT JOIN indexed_files ifx ON si.source_path = ifx.path + WHERE + (ifx.path IS NULL OR ifx.hash = ?) AND + (NOT EXISTS (SELECT 1 FROM template_matches WHERE source_path = si.source_path) OR + NOT EXISTS (SELECT 1 FROM correction_signals WHERE source_path = si.source_path) OR + NOT EXISTS (SELECT 1 FROM tool_events WHERE source_path = si.source_path AND extraction_version = 0)) + ORDER BY si.source_path +`, recoveredSourceHash) +``` + +Keep the existing stale-template merge, batching, idempotency predicates, and transaction boundaries unchanged. + +- [ ] **Step 8: Run marker-consumer and broader storage tests** + +Run: + +```bash +go test ./internal/storage -run 'Test(GetFileHashes|GetStats|SyncFiles|BackfillDerived|Purge|Validate)' -count=1 +``` + +Expected: PASS. In particular, existing `TestBackfillDerivedSkipsOnDiskFiles` must still prove that a real hash excludes an on-disk path from lossy backfill. + +- [ ] **Step 9: Commit Task 2** + +```bash +git add internal/storage/sync.go internal/storage/queries.go internal/storage/backfill.go internal/storage/storage_test.go internal/storage/backfill_test.go +git commit -m "fix(storage): distinguish recovered source accounting" +``` + +--- + +### Task 3: Add the CLI recovery-to-validation regression + +**Files:** +- Modify: `cmd/backscroll/recover_test.go:1-15, 145-225, 370-425` +- Read fixture: `tests/fixtures/recovery/active-v13.sql` +- Read fixture: `tests/fixtures/recovery/stranded-v7.sql` + +**Interfaces:** +- Consumes: the public `recover --from` and `validate --indexed-only` commands, supported v13/v7 fixture schemas, and recovery output's `backup path: ` field. +- Produces: a hermetic regression proving the installed recovery destination is valid, complete, FTS-queryable, and backed up byte-for-byte. + +- [ ] **Step 1: Add a fixture database helper** + +Add `database/sql` to imports. Add this helper near `createRecoverTestDB`: + +```go +func createRecoverFixtureDB(t *testing.T, path, fixture string, replacements map[string]string) { + t.Helper() + fixturePath := filepath.Join("..", "..", "tests", "fixtures", "recovery", fixture) + body, err := os.ReadFile(fixturePath) + if err != nil { + t.Fatalf("read recovery fixture %s: %v", fixturePath, err) + } + sqlText := string(body) + for from, to := range replacements { + sqlText = strings.ReplaceAll(sqlText, from, to) + } + db, err := sql.Open("sqlite", path) + if err != nil { + t.Fatalf("open recovery fixture %s: %v", path, err) + } + if _, err := db.Exec(sqlText); err != nil { + _ = db.Close() + t.Fatalf("execute recovery fixture %s: %v", fixture, err) + } + if err := db.Close(); err != nil { + t.Fatalf("close recovery fixture %s: %v", path, err) + } +} +``` + +The `storage` package imported by this test already registers the modernc SQLite driver. Do not modify fixture files on disk. + +- [ ] **Step 2: Add the end-to-end regression** + +Add `TestRecoverInstalledDestinationPassesIndexedValidation`: + +```go +func TestRecoverInstalledDestinationPassesIndexedValidation(t *testing.T) { + dir := t.TempDir() + home := filepath.Join(dir, "home") + if err := os.MkdirAll(home, 0o755); err != nil { + t.Fatal(err) + } + activePath := filepath.Join(dir, "active.db") + strandedPath := filepath.Join(dir, "stranded.db") + createRecoverFixtureDB(t, activePath, "active-v13.sql", map[string]string{ + "uuid-active-v13": "11111111-1111-4111-8111-111111111111", + }) + createRecoverFixtureDB(t, strandedPath, "stranded-v7.sql", map[string]string{ + "uuid-stranded-v7": "22222222-2222-4222-8222-222222222222", + }) + activeBefore, err := os.ReadFile(activePath) + if err != nil { + t.Fatal(err) + } + + t.Setenv("HOME", home) + t.Setenv("BACKSCROLL_CONFIG_DIR", filepath.Join(dir, "config")) + t.Setenv("BACKSCROLL_DATABASE_PATH", activePath) + t.Chdir(dir) + + var recoverOut, recoverErr bytes.Buffer + recoverCmd := buildRootCmd(&recoverOut, &recoverErr) + recoverCmd.SetArgs([]string{"recover", "--from", strandedPath}) + if err := recoverCmd.Execute(); err != nil { + t.Fatalf("recover: %v\nstderr=%s", err, recoverErr.String()) + } + if recoverErr.String() != "" { + t.Fatalf("recover stderr = %q, want empty", recoverErr.String()) + } + backupPath := recoverOutputValue(t, recoverOut.String(), "backup path: ") + backupBytes, err := os.ReadFile(backupPath) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(backupBytes, activeBefore) { + t.Fatal("active backup differs from pre-recovery database") + } + + var validateOut, validateErr bytes.Buffer + validateCmd := buildRootCmd(&validateOut, &validateErr) + validateCmd.SetArgs([]string{"validate", "--indexed-only"}) + if err := validateCmd.Execute(); err != nil { + t.Fatalf("validate recovered destination: %v\nstdout=%s\nstderr=%s", err, validateOut.String(), validateErr.String()) + } + if validateErr.String() != "" { + t.Fatalf("validate stderr = %q, want empty", validateErr.String()) + } + + db, err := storage.OpenReadOnly(activePath) + if err != nil { + t.Fatal(err) + } + defer func() { _ = db.Close() }() + var records, accountedPaths, ftsHits int + if err := db.DB().QueryRow(`SELECT COUNT(*) FROM search_items`).Scan(&records); err != nil { + t.Fatal(err) + } + if err := db.DB().QueryRow(`SELECT COUNT(*) FROM indexed_files`).Scan(&accountedPaths); err != nil { + t.Fatal(err) + } + if err := db.DB().QueryRow(`SELECT COUNT(*) FROM messages_fts WHERE messages_fts MATCH 'default'`).Scan(&ftsHits); err != nil { + t.Fatal(err) + } + if records != 4 || accountedPaths != 4 || ftsHits != 2 { + t.Fatalf("recovered database counts records=%d paths=%d fts=%d, want 4/4/2", records, accountedPaths, ftsHits) + } +} +``` + +The expected accounting count is four because the two fixtures contain four distinct `source_path` values. Retain command output in failure messages so a regression is diagnosable. + +- [ ] **Step 3: Run the end-to-end regression** + +Run: + +```bash +go test ./cmd/backscroll -run TestRecoverInstalledDestinationPassesIndexedValidation -count=1 +``` + +Expected: PASS. The RED behavior is already pinned by Task 1's failing destination-accounting tests: on the original implementation, recovery produced zero accounting rows and public validation reported four orphaned `search_items`. + +- [ ] **Step 4: Run all recovery and compatibility command tests** + +Run: + +```bash +go test ./cmd/backscroll -run 'TestRecover|TestValidateCurrentIndexIntegrityFailureIsGenericAndReadOnly' -count=1 +``` + +Expected: PASS. The existing genuine-orphan test must continue reporting `orphaned search_items`; recovery diagnostics and backup behavior must remain unchanged. + +- [ ] **Step 5: Commit Task 3** + +```bash +git add cmd/backscroll/recover_test.go +git commit -m "test(recovery): validate installed recovered index" +``` + +--- + +### Task 4: Run the complete quality gate and review the diff + +**Files:** +- Verify only; modify earlier files only if a failing gate reveals a defect. + +**Interfaces:** +- Consumes: all Task 1-3 commits. +- Produces: a verified implementation satisfying issue #35 and the repository's aggregate coverage gate. + +- [ ] **Step 1: Check formatting and static analysis** + +Run: + +```bash +just check +``` + +Expected: PASS (`gofmt --check` and `go vet`). If formatting fails, run `just fmt`, inspect the diff, and recommit only the formatting changes with the task that introduced them. + +- [ ] **Step 2: Run the full test suite** + +Run: + +```bash +just test +``` + +Expected: PASS for every package. + +- [ ] **Step 3: Run the release-blocking CI mirror** + +Run: + +```bash +just ci +``` + +Expected: build succeeds, scrubbed-HOME tests pass, race checks pass, and aggregate statement coverage is at least 85%. + +- [ ] **Step 4: Review the final diff against the spec** + +Run: + +```bash +git diff HEAD~3..HEAD --check +git diff HEAD~3..HEAD --stat +git log --oneline -4 +``` + +Confirm all of the following from tests and code: + +- recovery writes exact marked accounting with NULL `last_indexed`; +- validation still uses path presence and rejects genuine orphans; +- hashes and status exclude markers; +- backfill includes markers; +- normal sync replaces markers; +- purge uses its existing last-row cleanup; +- no migration, schema edit, JSONL write, or unrelated refactor was added. + +- [ ] **Step 5: Record any gate-only fix** + +If Step 1-4 required a code change, rerun the narrow failing test and all three quality commands, then commit with a precise conventional message such as: + +```bash +git add internal/storage/recovery_accounting.go internal/storage/recovery_destination.go internal/storage/recovery_destination_test.go internal/storage/sync.go internal/storage/queries.go internal/storage/backfill.go internal/storage/storage_test.go internal/storage/backfill_test.go cmd/backscroll/recover_test.go +git commit -m "fix(storage): preserve recovered accounting invariant" +``` + +If no change was required, do not create an empty commit. diff --git a/docs/superpowers/specs/2026-08-18-systemic-index-compatibility-design.md b/docs/superpowers/specs/2026-08-18-systemic-index-compatibility-design.md new file mode 100644 index 0000000..c727b01 --- /dev/null +++ b/docs/superpowers/specs/2026-08-18-systemic-index-compatibility-design.md @@ -0,0 +1,496 @@ +# Systemic Index Compatibility for Backscroll Issues #30–#33 + +**Status:** Approved design + +**Date:** 2026-08-18 + +**Issues:** #30, #31, #32, #33 + +## Decision and reviewer path + +Backscroll will resolve #30–#33 through one **narrow, stateless compatibility boundary** for legacy database and configuration representations. The boundary inspects inputs and returns canonical plans or typed diagnostics. It never writes, starts transactions, owns workflow state, applies business rules, or orchestrates commands. Separate consumers execute schema migration, manifest ingestion, stranded-database recovery, and shipped-asset validation. + +Review and deliver the change in this order: + +1. **Plan 1 — inspection and migration primitives:** confirm schema inspection is stateless, release coverage is hermetic, and migration snapshot/transaction/final verification is shape-safe. This safe intermediate slice does not block commands, print `recover`, or close #31. +2. **Plan 3 — recovery and policy activation:** confirm active-plus-one-stranded recovery is registered and functional before shared stale-index refusal, cached-fallback removal, diagnostic behavior, direct-read exemption proof, or executable continuations become user-facing. +3. **Plan 2 — manifest ingestion:** confirm manifest-only external sources and all-active preflight consume the already-active fail-closed policy, so no manifest failure can serve cached rows. +4. **Plan 4 — guidance:** confirm every shipped command resolves against the real Cobra tree, every command declares a known asset effect, and unannotated/unknown commands refuse harness execution. +5. **Closure evidence:** confirm every plan boundary is fully green with no skipped closure test; #31 closes only after Plan 2 completes both migration/policy and ingestion evidence. + +This design supersedes the earlier `open-issues-triage` conclusions where they differ. In particular, the user-facing command is `backscroll recover`; recovery replaces the active database with the verified union of its existing records and one stranded source, while remaining a recovery-only operation rather than an arbitrary cross-machine or multi-source merge. Legacy `[sources]` is rejected with an exact manifest example rather than merely warned about. + +## Problem statement + +Four reports expose three systemic root clusters rather than four independent patches. + +| Root cluster | Issues | Failure | Root correction | +|---|---|---|---| +| Index lineage compatibility | #31, #32 | Version rows do not fully describe historical schema shape; failed migration can leave an unusable or stale index; expired raw sessions make rebuild lossy. | Inspect real schema shape, migrate supported Go lineages transactionally, block stale data access, and provide atomic stranded-database recovery. | +| External-source contract | #31, #33 | Legacy `[sources]` parses without a production consumer; active manifests can declare an unregistered decoder and be skipped; the shipped decisions preset names a nonexistent command. | Make `*.inputs.toml` the only truth, register `markdown_sections`, preflight all active manifests, and reject invalid roots/manifests/decoders visibly. | +| Guidance and asset integrity | #30 | Shipped guidance can drift from executable CLI behavior and can encourage unsupported conclusions after weak recall evidence. | Validate every shipped consumer asset against the Cobra command tree and add the approved recall guardrails. | + +The reported mechanisms are not treated as authoritative. For example, glob discovery already exists, while `markdown_sections` registration and visible preflight are missing. Likewise, published v0.3.18 Go DDL contains `source_metadata`, but an observed failing database may not; actual shape, not the version label alone, decides compatibility. + +## Goals and non-goals + +### Goals + +- Upgrade every published Go schema lineage to the current schema without row loss. +- Diagnose unsupported, corrupt, or stale indexes with a non-empty executable continuation argv. +- Prevent all index-backed data commands from reading cached results after migration or sync leaves the index stale, while keeping direct file reads available and clearly non-evidentiary about index freshness. +- Keep `*.inputs.toml` manifests as the only external-source truth. +- Make Markdown decision files a normal reader input through `markdown_sections`. +- Recover the union of the configured active database and one stranded historical database into a fresh, verified active database, preserving recent active records and historical stranded records without mutating the stranded source. +- Account for every record from both recovery inputs and reject every identity conflict or uninterpretable record before replacement. +- Ensure shipped skills, presets, and distributed documentation name executable commands from the real Cobra tree. +- Close issues only with named behavior tests. + +### Non-goals + +- Arbitrary cross-machine merge, multiple stranded sources, synchronization, replication, or conflict resolution. The required active-plus-one-stranded recovery union is the sole merge-like operation. +- A cached-result fallback after migration or sync failure. +- Automatic mutation of `config.toml` or generation of manifests from `[sources]`. +- Dual ingestion from both `[sources]` and manifests. +- In-place migration of frozen Rust v0 databases. +- Repairing arbitrary SQLite corruption or unknown third-party schemas. +- Fixing the reported large-file index gap without a minimal reproducing fixture. +- Adding a second discovery pipeline, decoder plugin framework, migration DSL, workflow engine, or persistent compatibility state. +- Moving #30 into runtime compatibility logic; it is a shipped-asset validation concern. + +## Architecture + +```mermaid +flowchart LR + DB[(Historical, stranded, or active SQLite)] --> I[compat Inspector] + CFG[config.toml and *.inputs.toml] --> I + I -->|MigrationPlan| M[storage migration consumer] + I -->|ManifestPlan| G[manifest ingestion consumer] + I -->|RecoveryPlan| R[recover command consumer] + I -->|Diagnostic| C[command policy] + + M --> ACTIVE[(Active SQLite)] + G --> READERS[reader Registry] + READERS --> SYNC[SyncFiles] + SYNC --> ACTIVE + R --> TMP[(Fresh temporary SQLite)] + TMP --> SWAP[verified atomic replacement] + SWAP --> ACTIVE + + ASSETS[skills, presets, distributed docs] --> AV[asset command validator] + COBRA[real Cobra command tree] --> AV + + C --> DATA[index-backed commands: block if stale] + C --> DIAG[status / validate: report + continuation] +``` + +### Responsibilities + +| Component | Owns | Must not own | +|---|---|---| +| `internal/compat` | Read-only shape/config inspection; canonical plans; typed diagnostics; deterministic compatibility classification. | Writes, transactions, backups, temporary paths, atomic rename, Cobra output, retries, workflow state, business rules. | +| Storage migration consumer | Snapshot, transaction, ordered migration execution, post-migration verification. | Guessing lineage from version alone or swallowing a compatibility diagnostic. | +| Manifest ingestion consumer | Manifest loading, registry preflight, discovery, decoding, sync transaction, stale-state propagation. | Reading legacy `[sources]`, silently skipping inputs, or maintaining a second source model. | +| Recovery consumer | Resolve and de-duplicate active/stranded paths; open both inputs read-only; request a union plan; create a fresh temporary destination; import, validate, back up active, and atomically replace it. | Arbitrary cross-machine or multi-source merge semantics, partial import, stranded-source mutation, or conflict resolution. | +| Command policy | Classifying data versus diagnostic commands and enforcing blocking behavior. | Cached fallback or command-specific reinterpretation of diagnostics. | +| Asset command validator | Discovering consumer `.md`, `.toml`, and `.sh` assets under approved distribution roots; extracting commands; requiring a known effect on every root constructor; resolving/exercising against `buildRootCmd`. | Runtime index compatibility, a hand-maintained verb allowlist, or a default effect for unannotated commands. | + +## Typed compatibility model + +The model is intentionally small and introduced only when a real consumer needs it. Plan 1 owns only schema inspection and migration primitives: + +```go +// internal/compat, introduced by Plan 1 +package compat + +type Code string + +const ( + CodeUnsupportedLineage Code = "unsupported_lineage" + CodeMigrationFailed Code = "migration_failed" + CodeIndexStale Code = "index_stale" +) + +type Diagnostic struct { + Code Code + Summary string + Continuation []string +} + +type SchemaShape struct { + AppliedVersion int + Signature string +} + +type MigrationStep struct { Version int; Name string } +type MigrationPlan struct { From SchemaShape; Steps []MigrationStep } + +type Queryer interface { + QueryContext(context.Context, string, ...any) (*sql.Rows, error) + QueryRowContext(context.Context, string, ...any) *sql.Row +} + +func InspectIndex(ctx context.Context, q Queryer) (MigrationPlan, *Diagnostic, error) +func VerifyCurrentShape(ctx context.Context, q Queryer) error +``` + +Plan 1 diagnostics are internal inspection results. They are not rendered by commands and may have an empty `Continuation`; Plan 3 fills executable argv only after `recover` exists. + +```go +// introduced by Plan 3 when recovery consumes them +const ( + CodeRecoveryConflict Code = "recovery_conflict" + CodeUninterpretableRow Code = "uninterpretable_row" +) + +type RecoveryInput struct { + Shape SchemaShape + Records []models.IndexedRecord + RowCount int +} + +type CanonicalRecord struct { + Record models.IndexedRecord + PayloadHash string +} + +type RecoveryPlan struct { + InputShapes []SchemaShape + Records []CanonicalRecord + ExactDuplicates int +} + +func PlanRecovery(inputs []RecoveryInput) (RecoveryPlan, []Diagnostic, error) +``` + +Plan 3 moves only the canonical `IndexedRecord` representation from storage to `internal/models` to avoid the real storage→compat→storage import cycle. `IndexedRecordQuery` and SQL query ownership remain in storage. + +```go +// introduced by Plan 2 when manifest preflight consumes them +const ( + CodeLegacySources Code = "legacy_sources" + CodeInvalidManifest Code = "invalid_manifest" + CodeUnknownDecoder Code = "unknown_decoder" + CodeInvalidRoot Code = "invalid_root" +) + +type ReaderLookup interface { + ForDef(input_config.InputDefinition) (readers.SessionReader, error) +} + +type ManifestPlan struct { Inputs []ResolvedInput } +type ResolvedInput struct { + Definition input_config.InputDefinition + ReaderName string +} + +func InspectManifests(defs []input_config.LoadedDefinition, registry ReaderLookup) (ManifestPlan, *Diagnostic, error) +func InspectLegacyConfig(raw []byte) *Diagnostic +``` + +The recovery consumer resolves paths first and passes active then stranded as logical inputs; when `--from` resolves to active, it passes one input so rows are not scanned or counted twice. Go `error` represents inability to inspect; typed diagnostics represent observed incompatibility. Once Plan 3 activates command policy, every user-facing block carries non-empty argv that resolves through Cobra. Plans contain identifiers and ordered actions, not SQL strings, handles, callbacks, destination paths, or mutable status. + +## Detailed flows + +### Normal migration + +1. Resolve the configured active database path. +2. Open the database for inspection and read `schema_migrations`, `sqlite_master`, `PRAGMA table_info`, index metadata, and relevant trigger definitions. +3. Match the observed signature to the published Go lineage catalog. A version row is evidence, not the sole selector. +4. If no destructive step is required, execute the returned steps in one transaction and verify the final shape before commit. +5. Before the first destructive step, create and fsync a snapshot beside the active database. Snapshot creation is a consumer responsibility and occurs before the migration transaction. +6. Execute every migration step and final-shape verification in one transaction. Any execution or verification failure rolls back every step. +7. Reopen or re-inspect the committed database. Only then may ingestion and an index-backed command continue. + +A migration that is already current returns an empty plan. Re-running inspection is idempotent. + +### Incompatible or stale index + +1. If inspection cannot map the shape safely, Plan 1 returns internal `unsupported_lineage` without command text. After Plan 3 registers and completes `recover`, the active command policy renders non-empty argv `backscroll recover --from --dry-run`; diagnostics never print a literal placeholder. +2. If snapshot, migration, verification, manifest preflight, decoding, discovery, or sync fails after the current query corpus may have become stale, classify the active index as unusable for index-backed commands in that invocation. +3. Stop before index-backed search, list, pattern output, mutation, cached success output, or zero-result claims. +4. `backscroll read ` remains available because it reads the supplied file directly through `internal/reader`; it must never be described or tested as proof that the index is current. +5. `status` and `validate` remain available. They inspect read-only, print the primary diagnostic and any safely collectible secondary diagnostics, and print a non-empty executable continuation argv. +6. A later index-backed command performs a fresh inspection. No persistent compatibility state or cached failure flag is introduced. + +### Manifest ingestion + +1. Parse all installed `*.inputs.toml` files. Invalid TOML, duplicate IDs, unsupported versions, missing required fields, invalid roots, and unsafe/unknown discovery patterns are visible failures. +2. Reject any non-empty legacy `[sources]` before ingestion. Do not mutate the config and do not ingest either representation. +3. Resolve every active manifest through `readers.Registry` before discovering or syncing any file. This is an all-input preflight: one unknown decoder blocks the whole sync. +4. Register `markdown_sections` as a normal `SessionReader`. It discovers through the existing manifest discovery path, hashes deterministically, emits ordered sections, and uses whole-document fallback when no eligible section headings exist. +5. Only after all active definitions pass preflight, discover all files. An invalid or inaccessible declared root blocks sync visibly; an existing valid root with no matching files is a valid empty input. +6. Parse and validate all planned files, then invoke existing sync execution. Any failure blocks index-backed commands; no partially refreshed corpus may be presented as current. Direct file `read` remains available without making an index-freshness claim. + +Legacy configuration is rejected with an exact conversion example. For example: + +```toml +# Rejected legacy config.toml +[sources] +decisions = ["/work/project/docs/decisions"] +``` + +becomes a separate file such as `decisions.inputs.toml`: + +```toml +version = 1 + +[[inputs]] +id = "decisions" +source = "decision" +active = true + +[inputs.discover] +roots = ["/work/project/docs/decisions"] +include = ["**/*.md"] +exclude = ["TEMPLATE.md", "template.md", "Template.md"] +follow_symlinks = false + +[inputs.decode] +format = "markdown_sections" +``` + +The diagnostic includes the actual legacy category and paths in this template. It never writes the file automatically. + +### Recovery and dry-run + +The command shape is: + +```text +backscroll recover --from [--dry-run] +``` + +The destination is the configured active database. Recovery builds the union of that active database and exactly one stranded database supplied by `--from`; it is not a general `--into`, cross-machine, or multi-source merge tool. + +1. Resolve and canonicalize the configured active path and `--from` path. If they resolve to the same database, treat them as one logical input rather than scanning, counting, or importing it twice. +2. Open every distinct input with SQLite `mode=ro`. Each logical input is read-only during planning; stranded bytes and sidecars must remain unchanged throughout recovery. +3. Inspect each input lineage and adapt or explicitly reject every row from both inputs into the current canonical record shape. +4. Build one complete union identity/equivalence/conflict plan in memory or bounded read-only batches. Exact duplicate identities with equivalent canonical payloads collapse to one record. The same identity with different payloads is a conflict. Any conflict or uninterpretable row in either input makes the entire plan non-applicable. +5. With `--dry-run`, print the same per-input and union counts, duplicate identities, conflicts, uninterpretable records, input shapes, and intended replacement path that apply would use. Use the identical union planner and create no destination, temporary database, backup, journal, or other file. +6. Without `--dry-run`, stop immediately if planning does not account for every input row or contains any conflict or uninterpretable row. +7. Create a fresh temporary database in the active database's directory so final rename is on the same filesystem. Initialize it at the current schema; never append into the existing active database. +8. Import the verified union in one transaction and validate source accounting, union row counts, identities, foreign-key integrity, FTS consistency, schema shape, and representative queryability before commit. +9. Close all input and destination handles. Preserve the original active database as a uniquely named backup and fsync the containing directory. +10. Atomically rename the independently verified temporary database into the active path and fsync the directory again. +11. Report the active path, backup path, per-input counts, exact duplicates collapsed, and final union count. Never delete the backup automatically. + +When `--from` resolves to the active path, the single logical input still follows the fresh-destination and backup flow. A failure before final replacement leaves the original active database untouched; a failure during replacement restores or retains a valid original path according to the platform-specific atomic replacement helper. + +## Error precedence and command behavior + +### Precedence + +When more than one problem is observable, commands select the primary diagnostic in this order: + +1. **Invocation/config cannot be interpreted:** invalid CLI arguments, unreadable config, invalid TOML. +2. **Parallel source truth:** non-empty legacy `[sources]`. +3. **Index cannot be trusted:** unreadable/corrupt database, unsupported lineage, failed migration, failed final-shape verification. +4. **Manifest set cannot be trusted:** unsupported manifest version, duplicate ID, unknown decoder, invalid or inaccessible root, invalid pattern. +5. **Refresh cannot be trusted:** discovery, decode, or sync failure; index is stale for this invocation. +6. **Requested operation failure:** query, read, mutation, output, or recovery execution failure. + +Within one level, preserve deterministic manifest-file and input-ID order. `validate` may report all independently inspectable diagnostics, but its first item and exit classification follow this precedence. A wrong continuation is worse than a missing one, and missing is forbidden for a block: tests assert every blocking diagnostic has a non-empty argv and execute every printed continuation shape against the Cobra tree. This proves invocation, not guaranteed successful remediation. + +### Command classes + +| Class | Commands/surfaces | Contract when index or ingestion is incompatible/stale | +|---|---|---| +| Index-backed data reads | `search`, `list`, `patterns` | Exit non-zero before reading or printing cached rows. Print the primary typed diagnostic and non-empty executable continuation. | +| Direct file reads | `read` | Remain available because the supplied file is read directly through `internal/reader`. Output is file-decoder output only and is never presented as evidence that the index is current. | +| Data mutations | `rebuild`, `purge`, `annotate`, and automatic sync preceding another command | Exit non-zero before mutation or success output. No partial migration or sync is represented as success. | +| Diagnostics | `status`, `validate` | Remain runnable, inspect read-only, report detected shape/staleness and a non-empty executable continuation, and exit non-zero when unhealthy. They do not migrate or sync merely to diagnose. | +| Remediation | `recover` | May inspect incompatible active/stranded inputs and write only through the atomic union-recovery flow. `--dry-run` writes nothing. | +| Configuration/help | `config`, `help`, `version`, shell completion | Remain available without opening or trusting the index. | + +No index-backed command serves cached results after a migration or sync failure. `--indexed-only`, robot/JSON output, source filters, or other flags do not bypass the block. Direct `read` remains available but carries no index-freshness claim. Machine-readable modes carry the same diagnostic code and non-empty executable continuation argv as human output. + +## Recovery identity, conflicts, and atomicity + +### Identity and equivalence + +| Record state in either input | Identity key | Union comparison | Outcome | +|---|---|---|---| +| Non-empty valid UUID | UUID | Compare canonical payload hash for every active/stranded record sharing UUID. | Collapse exact duplicates to one union record; conflict if payload differs. Do not fall back to path/ordinal. | +| Empty UUID with valid source path and non-negative ordinal | Exact `(source_path, ordinal)` | Compare canonical payload hash for every active/stranded record sharing the pair. | Collapse exact duplicates to one union record; conflict if payload differs. | +| Same content hash under different identities | Each declared identity remains distinct. | Hash proves payload equivalence only. | Preserve both records; content hash never deduplicates identities. | +| Missing/invalid UUID and unusable path or ordinal | No safe identity. | Not applicable. | Uninterpretable; abort the entire union recovery. | +| Historical row cannot map required canonical data | Identity may or may not exist. | Not applicable. | Uninterpretable; abort the entire union recovery. | + +`source_path` is compared exactly as stored; recovery does not normalize case, separators, symlinks, hosts, or project roots. UUID validity uses the formats actually published by Backscroll readers; it does not invent UUIDs for legacy rows. + +### Atomicity invariants + +- Every distinct active/stranded input is opened read-only for planning and is never migrated, vacuumed, journal-mode changed, or otherwise mutated; the stranded source remains unchanged throughout recovery. +- Planning accounts for every row from both inputs as importable, an exact duplicate, conflicting, or uninterpretable. If both paths resolve identically, one logical input is accounted once. +- The verified output is the union: recent active-only records and historical stranded-only records are both preserved; exact duplicate identities collapse. +- Any same-identity payload conflict or uninterpretable record aborts the entire recovery. There is no `--force`, `--skip`, or partial mode. +- Apply and dry-run use the same union planner and therefore produce the same plan for the same input bytes and active-path resolution. +- The temporary destination starts empty at the current schema; recovery imports the planned union and never appends into the existing active database. +- All destination inserts and database-level verification occur in one transaction. +- Replacement occurs only after transaction commit, handle closure, and independent read-only verification of the temporary destination. +- The temporary destination is created beside the active database; atomic rename never crosses filesystems. +- The original active database is preserved as a backup before replacement and is never automatically deleted. +- A failed recovery leaves either the original active path intact or a restorable backup plus an explicit diagnostic; it never reports success without the verified destination at the active path. + +## Published lineage compatibility strategy + +Compatibility is maintained by **unique observed schema shape**, with release ranges supplying the required corpus. The current repository has Go releases beginning at v0.3.7 and migration history through V13. + +| Published release range | Language | Highest recorded migration | Required fixture strategy | +|---|---|---:|---| +| v0.1.9–v0.3.6 | Rust | Not applicable | Outside in-place migration. `recover --dry-run` may recognize explicitly supported extractable shapes; otherwise report the frozen Rust v0 boundary. | +| v0.3.7–v0.3.10 | Go | V1 | Fixture every unique V1 shape/signature from published DDL or release-produced database. | +| v0.3.11–v1.3.5 | Go | V3 | Cover V2/V3 embedding-table and column shape variants. Include the observed v0.3.18-style missing-`source_metadata` variant even though published DDL contains the column. | +| v1.4.0–v1.4.4 | Go | V4 | Cover split FTS tables, triggers, and repopulated indexes. | +| v2.0.0–v2.1.0 | Go | V5 | Cover removal of `session_events` while `source_metadata` may still vary by observed shape. | +| v2.2.0–v2.2.3 | Go | V7 | Cover conditional V6 column removal and V7 reasoning triggers. | +| v2.3.0 | Go | V8 | Cover perennity fields and `tool_events`. | +| v2.4.0–v2.5.0 | Go | V9 | Cover tool-event UUID uniqueness. | +| v2.6.0 | Go | V10 | Cover template-mining tables. | +| v2.7.0 | Go | V11 | Cover correction signals. | +| v2.8.0–v2.11.0 | Go | V12 | Cover annotations. | +| v2.12.0–v3.2.5 | Go | V13 | Cover backfill discovery indexes and current-shape idempotency through the latest confirmed published release. | + +A checked-in release-to-schema manifest is the hermetic inventory of published Go releases from v0.3.7 through v3.2.5. It maps each release to a checked-in schema fixture/signature and records the source checksum or provenance needed to reproduce that classification. Multiple releases may share one fixture only when their relevant schema signatures are identical. Unit and CI tests consume only this checked-in manifest and fixtures; they do not depend on ambient/local `git tag` state or network access. The currently available local tags ending at v2.16.1 therefore cannot silently truncate coverage of the confirmed v3.0.0–v3.2.5 releases. A deliberate maintainer generation/maintenance check may compare the manifest with GitHub releases and fail when a published release is unmapped, but it is separate from hermetic tests. Version rows, tables, columns, indexes, and relevant triggers contribute to classification; irrelevant SQLite metadata does not. + +Rust v0 is a format boundary, not a hidden migration branch. No Rust database is modified in place. An explicit recovery adapter may be added only for a verified Rust shape and remains subject to the same identity, conflict, dry-run, and atomic replacement rules. + +## Test strategy and closure evidence + +Tests use table-driven cases, `t.TempDir()`, immutable fixture copies, and focused public boundaries. Historical fixtures must come from published DDL/release binaries or documented observed shapes; tests must not teach fixtures to match current assumptions. + +| Test | Behavior proved | Issue evidence | +|---|---|---| +| `TestCheckedInReleaseSchemaManifestIsComplete` | The checked-in v0.3.7–v3.2.5 release inventory maps every listed Go release to an existing checked-in fixture/signature without consulting local tags or the network. | #31 migration half | +| `TestPublishedGoLineagesUpgradeLosslessly` | Every manifest-mapped published Go shape reaches current schema with rows and FTS queryability preserved. | #31 migration half | +| `TestHistoricalLineageWithoutSourceMetadataUpgradesLosslessly` | The observed divergent shape does not fail V6 and loses no data. | #31 original upgrade symptom | +| `TestMigrationSnapshotAndRollbackOnDestructiveFailure` | Snapshot precedes destructive work; injected failure rolls back all schema/data changes. | #31 safety | +| `TestStaleIndexBlocksIndexBackedCommands` | Table-driven `search/list/patterns/rebuild/purge/annotate` cases return no cached data or mutation after migration/sync failure. | #31 blocking contract | +| `TestDirectReadRemainsAvailableButClaimsNoIndexFreshness` | `read` decodes the supplied file through `internal/reader` while the index is stale, and its output contains no index-current claim. | #31 operability boundary | +| `TestBlockingDiagnosticsHaveExecutableContinuations` | Every blocking diagnostic, including `status` and `validate` output for an unhealthy index, has non-empty continuation argv that resolves and executes through Cobra; success of the remediation itself is not assumed. | #31 operability | +| `TestLegacySourcesRejectedWithExactManifestExample` | `[sources]` causes no ingestion or config mutation and prints a complete equivalent manifest. | #31 ingestion half | +| `TestActiveManifestsPreflightBeforeSync` | Unknown decoder, invalid manifest, duplicate ID, or invalid root blocks before any input syncs. | #31, #33 | +| `TestDecisionManifestMarkdownSectionsEndToEnd` | Nested and newly added Markdown files are discovered, decoded, indexed, and retrieved with source `decision`. | #31 ingestion half, #33 | +| `TestMarkdownSectionsReaderSectionsAndWholeDocument` | Ordered section parsing and no-heading fallback are deterministic. | #33 | +| `TestRecoverDryRunMatchesUnionApplyPlanWithoutWrites` | Dry-run and apply use the identical active-plus-stranded union plan; dry-run creates or changes no files. | #32 | +| `TestRecoverUnionPreservesActiveAndStrandedRecords` | Active-only recent records and stranded-only historical records both appear in the fresh current-schema destination. | #32 | +| `TestRecoverIdentityAndConflictMatrixAcrossInputs` | UUID precedence, path/ordinal fallback, exact-duplicate collapse, hash-only equivalence, and same-identity conflicts across either input follow the table. | #32 | +| `TestRecoverConflictOrUninterpretableRollsBackEverything` | One bad row in either input prevents every import and leaves active/stranded bytes unchanged. | #32 | +| `TestRecoverSameResolvedPathIsOneInput` | `--from` resolving to the active database scans, accounts, and imports each record once. | #32 | +| `TestRecoverAtomicallyReplacesAndPreservesActiveBackup` | A fresh verified union destination replaces active atomically and the original active backup remains. | #32 | +| `TestRecoverStrandedSourceIsReadOnly` | Stranded database and sidecar metadata are byte/mtime stable across dry-run, success, and failure. | #32 | +| `TestEveryRootCommandDeclaresAssetEffect` | Root and every direct Cobra command declare exactly one known `read`, `write`, or `replace` effect; missing/unknown annotations fail. | #30 harness safety | +| `TestAssetHarnessRejectsMissingOrUnknownEffect` | The harness refuses execution before setup or command dispatch when effect metadata is absent or unknown. | #30 harness safety | +| `TestShippedConsumerAssetDiscovery` | Discovery includes shipped consumer `.md`, `.toml`, and `.sh`, including `.claude/skills/backscroll-doctor/assets/gather.sh`, while excluding only the three history trees. | #30 surface accountability | +| `TestAllShippedBackscrollCommandsAreExecutable` | Commands extracted from every discovered shipped asset resolve against `buildRootCmd` and pass the fail-closed safe harness. | #30, stale #33 preset | +| `TestShippedSearchGuidanceGuardrails` | Guidance requires ranked-hit inspection, artifact vocabulary, malformed-call handling, and explicit index-gap reproduction. | #30 | + +Focused tests run before package or repository-wide suites. Integration-style CLI tests use temporary config/database roots and never a real home directory. The command-asset test reports asset path, line, parsed argv, and Cobra failure so drift is directly repairable. + +### Issue closure gates + +- **#31 closes only after** `TestCheckedInReleaseSchemaManifestIsComplete`, `TestPublishedGoLineagesUpgradeLosslessly` (including the missing-column fixture and V13 coverage through v3.2.5), the index-backed blocking/direct-read boundary tests, **and** the ingestion set (`TestLegacySourcesRejectedWithExactManifestExample`, `TestActiveManifestsPreflightBeforeSync`, and `TestDecisionManifestMarkdownSectionsEndToEnd`) pass. +- **#33 shares ingestion evidence** and closes only when the end-to-end decision retrieval and preflight tests pass; existing glob unit tests alone are insufficient. +- **#32 requires active-plus-stranded union fixtures**, including preservation of both record sets, exact-duplicate collapse, cross-input conflict/uninterpretable rollback, same-path de-duplication, dry-run parity, stranded-source immutability, active-backup preservation, and atomic replacement verification. +- **#30 requires complete asset discovery, explicit known effects on every root command, fail-closed rejection of missing/unknown effects, all consumer-facing commands to be executable**, not merely present as strings, plus the four guidance guardrails. + +## Four reviewable delivery slices + +| Slice | Scope and dependency | Review evidence | Rollback boundary | Net delta direction | +|---|---|---|---|---| +| 1. Inspector and migration primitives | Introduce schema-only compatibility types, checked-in release/schema catalog through v3.2.5, inspection, snapshot, one-transaction migration, and final verification. Do not activate command refusal or print `recover`. | Hermetic release inventory, published-lineage, divergent-shape, snapshot, rollback, and final-shape tests; no skip and no Cobra dependency. | Revert compatibility schema primitives, migration consumer changes, and fixture manifest together; leave existing DB backups untouched. | Initially positive because fixtures are new; no speculative manifest/recovery types or command-policy branches. | +| 3. Recover and blocking-policy activation | Depends on Plan 1. Introduce recovery types and canonical record ownership when consumed; build/register functional active-plus-one-stranded recovery; then activate shared index blocking, remove cached fallback, update all indexed consumers/diagnostics, and prove direct `read` remains exempt without changing it. | Full recovery invariants plus stale-index blocking, direct-read boundary, and executable-continuation tests, all passing with no skip. | Revert recover, canonical record move, policy/consumer changes, and tests as one chained slice; Plan 1 migration primitives remain safe. | Positive but narrow: one recovery path and one centralized policy; no general merge framework, flags, or persistent state. | +| 2. Manifests and Markdown reader | Depends on Plan 3’s active blocking policy. Introduce manifest-only types when consumed, reject `[sources]`, preflight every active manifest, register `markdown_sections`, and repair preset instructions. | Legacy rejection, all-input preflight, Markdown reader, end-to-end decision retrieval, and the complete no-skip #31 gate. | Revert reader registration, manifest preflight, legacy rejection, preset, and tests; Plan 3 continues to block other stale failures safely. | Neutral to negative in production code: remove/retire legacy source plumbing and silent skips; test/fixture lines increase. | +| 4. Guidance and asset command validation | Depends on final commands and repaired presets from Plans 1, 3, and 2. Discover shipped `.md`, `.toml`, and `.sh`; require explicit known effects on every root constructor; execute through a fail-closed hermetic harness. | Full shipped-asset command execution, effect-completeness/refusal tests, and four guardrail tests. | Revert guidance text, command metadata, and validator/tests together; runtime compatibility remains intact. | Neutral to negative in shipped prose; small explicit metadata/test growth prevents future drift. | + +Each slice is independently reviewable and revertible. Tests ship with the behavior they prove. Net line count is reported honestly; fixture growth is not disguised, and production-code growth must be justified by deletion of duplicated branches or by an explicit new recovery capability. + +## Risks and anti-overengineering constraints + +| Risk | Mitigation | +|---|---| +| Version labels hide divergent shapes. | Match shape signatures and retain observed variants; never use migration number alone. | +| A migration backup exists but is unusable. | Reopen and validate snapshots in tests before destructive steps. | +| Compatibility becomes an orchestration framework. | Keep the package stateless and read-only; plans contain identifiers, not SQL, handles, callbacks, paths, or mutable status. | +| Recovery evolves into general merge/sync. | Permit only the configured active database plus one `--from` stranded source, with no `--into`, additional sources, cross-machine workflow, or conflict-resolution flags. | +| Recovery drops recent active records while restoring history. | Plan every row from both read-only inputs and import only the verified union into a fresh current-schema destination. | +| Identity collapses distinct history. | UUID first, exact path/ordinal fallback, hash only for equivalence, exact-identity duplicates collapse, and conflicting identities abort. | +| Release coverage follows a stale clone or flaky network. | Make the checked-in release/schema manifest and fixtures authoritative for hermetic tests; isolate optional GitHub comparison as deliberate maintenance. | +| Legacy and manifest inputs drift. | Reject `[sources]`; never auto-mutate or dual-ingest. | +| One bad manifest silently starves another source. | Preflight every active manifest before any sync and block visibly. | +| Diagnostic commands accidentally mutate. | Open read-only and prohibit migration/sync in `status`/`validate` health inspection. | +| Asset validation becomes a brittle text allowlist. | Discover approved `.md`/`.toml`/`.sh` roots, resolve parsed command paths and flags against the constructed Cobra tree, and exercise safe invocations. | +| A future command executes under an unsafe default effect. | Define only `read`, `write`, and `replace`; annotate every root constructor; fail completeness and harness execution for missing/unknown metadata. | +| The design grows fields for hypothetical lineages. | Add a plan field only when a consumer needs it for an approved flow and a fixture proves the need. | + +Explicit constraints: + +- No new daemon, cache, state file, compatibility database, state machine, plugin protocol, migration language, retry loop, or feature flag. +- No `--force`, `--skip-conflicts`, `--merge`, `--partial`, or cached-result escape hatch. +- No duplicate manifest or reader abstractions when `input_config` and `readers.Registry` already provide the boundary. +- No issue-specific branches where one typed diagnostic and one policy table cover the root class. +- No implementation for the large-file symptom until a focused fixture fails on current code. + +## Alternatives considered + +### Root-batched, layer-free design + +This approach would patch each root directly in existing packages: conditional migration code in storage, decoder registration in command helpers, recovery code in the command, and documentation tests beside Cobra tests. + +It minimizes new types and may produce fewer initial lines. It was rejected because the same legacy-shape and diagnostic decisions would be repeated across migration, ingestion, and recovery consumers. That repetition makes error precedence drift likely. The selected compatibility boundary retains the root-batched delivery order but centralizes only inspection and translation; it does not add orchestration layers. + +### Issue-by-issue patches + +This approach would add guidance for #30, make V6 conditional for #31, add an import/merge command for #32, and add new glob handling for #33. + +It was rejected because it follows reported mechanisms rather than reproduced roots. It would duplicate glob discovery, leave `markdown_sections` and silent preflight failures unresolved, risk closing only half of #31, and create a general merge surface beyond stranded recovery. Four issues do not justify four unrelated patches when two root clusters are shared. + +### Dual `[sources]` and manifest ingestion + +Rejected because it creates two representations of external-source truth, ambiguous precedence, and permanent migration burden. An exact rejection example is simpler and safer than automatic mutation. + +### Cached reads with warnings + +Rejected because stale indexed results look authoritative and can produce false absence claims. Diagnostics and direct file `read` remain available; stale indexed data does not. + +### Source-only replacement, in-place recovery, or partial import + +Source-only replacement is rejected because it would discard recent records that exist only in the configured active database. In-place mutation removes the safest rollback boundary, and partial success makes accounting non-deterministic. The selected flow reads active plus one stranded source, constructs a fresh current-schema database from their verified union, preserves the original active backup, never mutates the stranded source, and atomically replaces only after complete verification. + +## Acceptance checklist + +### Architecture and behavior + +- [ ] `internal/compat` is stateless and performs no writes, transactions, orchestration, backups, or path replacement. +- [ ] Migration, ingestion, recovery, and asset validation remain separate consumers. +- [ ] Actual schema shape, not only migration rows, selects a plan. +- [ ] Destructive migration snapshots first, then executes and verifies in one transaction. +- [ ] The checked-in release/schema manifest maps every published Go release through v3.2.5 to a tested signature, and hermetic tests require neither local tags nor network access. +- [ ] Frozen Rust v0 is outside in-place migration. +- [ ] Index-backed data commands block after incompatible migration or stale sync; no flag enables cached fallback. +- [ ] Direct `read` remains available through `internal/reader` and is never presented as proof that the index is current. +- [ ] Every blocking diagnostic, including `status` and `validate`, carries a non-empty executable continuation argv; invocation does not imply guaranteed remediation success. +- [ ] `*.inputs.toml` is the only external-source truth. +- [ ] Legacy `[sources]` is rejected with an exact manifest example and no automatic mutation. +- [ ] `markdown_sections` is a registered normal reader. +- [ ] Every active manifest is preflighted before any sync begins. +- [ ] Unknown decoder, invalid manifest, or invalid/inaccessible declared root blocks visibly. +- [ ] Recovery resolves the configured active database plus one stranded `--from` source, treats an identical resolved path as one input, and opens every distinct input read-only for planning. +- [ ] Recovery accounts for every row from both inputs, preserves active-only and stranded-only records, and collapses only exact duplicate identities with equivalent payloads. +- [ ] Recovery uses UUID first, exact `(source_path, ordinal)` fallback, and content hash only for equivalence. +- [ ] Any same-identity conflict or uninterpretable record in either input aborts and rolls back everything. +- [ ] Recovery builds a fresh current-schema destination, verifies the union, atomically replaces active, preserves the original active backup, and never mutates the stranded source. +- [ ] `--dry-run` uses the same union planner and performs no writes. +- [ ] Shipped consumer discovery includes `.md`, `.toml`, and `.sh`, including `.claude/skills/backscroll-doctor/assets/gather.sh`, and excludes only `docs/roadmap`, `docs/research`, and `docs/superpowers`. +- [ ] Root and every current root command constructor explicitly declares one canonical `read`, `write`, or `replace` effect; the harness has no default and refuses missing/unknown values. +- [ ] Every command in discovered shipped skills, presets, scripts, and distributed docs is executable against the real Cobra tree. +- [ ] The large-file symptom remains out of implementation scope until reproduced. + +### Issue closure matrix + +| Issue | Required closure evidence | Must remain open when | +|---|---|---| +| #30 | `TestShippedConsumerAssetDiscovery`, `TestEveryRootCommandDeclaresAssetEffect`, `TestAssetHarnessRejectsMissingOrUnknownEffect`, `TestAllShippedBackscrollCommandsAreExecutable`, and `TestShippedSearchGuidanceGuardrails` pass across all shipped consumer assets. | Any shipped extension/root is omitted, any command effect is missing/unknown/defaulted, any invocation is merely string-matched rather than executable, or any guardrail is absent. | +| #31 | Hermetic checked-in release inventory through v3.2.5, published-lineage/divergent-shape migration, index-backed blocking, direct-read boundary, executable-continuation, legacy rejection, manifest preflight, and Markdown ingestion end-to-end tests pass. | Only migration or ingestion is fixed, `read` is incorrectly blocked, a blocking argv is empty/non-executable, or release coverage depends on local tags/network. | +| #32 | Active-plus-stranded union preservation, exact-duplicate collapse, same-path de-duplication, cross-input identity/conflict handling, dry-run parity, stranded-source immutability, full rollback, active-backup preservation, and atomic replacement fixtures pass. | Recovery drops either input's unique records, can skip/partially import, mutates stranded source, duplicates the same resolved input, or expands into arbitrary database merge. | +| #33 | `TestActiveManifestsPreflightBeforeSync` and `TestDecisionManifestMarkdownSectionsEndToEnd` pass, sharing #31 ingestion evidence. | Only existing glob discovery is demonstrated or the preset still names a nonexistent command. | diff --git a/docs/superpowers/specs/2026-08-19-recovered-source-accounting-design.md b/docs/superpowers/specs/2026-08-19-recovered-source-accounting-design.md new file mode 100644 index 0000000..8eb9e72 --- /dev/null +++ b/docs/superpowers/specs/2026-08-19-recovered-source-accounting-design.md @@ -0,0 +1,180 @@ +# Recovered Source Accounting Design + +**Date:** 2026-08-19 +**Issue:** [#35 — recovered index immediately fails validate as orphaned](https://github.com/pablontiv/backscroll/issues/35) + +## Summary + +`backscroll recover` currently installs the canonical union of records from the active and stranded databases while leaving `indexed_files` empty. The resulting database passes recovery's private verifier but immediately fails the public `validate --indexed-only` orphan check. + +Recovery will add explicit provisional source accounting to `indexed_files`. Each distinct `search_items.source_path` in the recovery plan will receive the reserved marker `backscroll:recovered` instead of a content hash. Consumers will distinguish this marker from a real SHA-256: validation will accept the accounted path, derived-data backfill will continue treating it as recoverable stored data, and autosync will not treat it as an unchanged source. A later real sync will replace the marker with the source file's actual hash. + +This change reconstructs database accounting only. It never creates, restores, or modifies session JSONL files. + +## Goals + +- Make a successfully recovered destination pass `validate --indexed-only` immediately. +- Preserve detection of genuine orphaned `search_items` rows. +- Represent unknown post-recovery source state explicitly without inventing a SHA-256. +- Keep recovered rows eligible for `BackfillDerived()` when their original sources are absent. +- Force a source that does exist on disk to be parsed on the next sync. +- Preserve canonical rows, FTS queryability, atomic installation, and the byte-identical active backup guarantee. + +## Non-goals + +- Recovering, recreating, or modifying original JSONL or Markdown source files. +- Preserving permanent historical provenance after a source has been synced normally. +- Introducing a new schema table or migration. +- Relaxing public validation for unaccounted rows. +- Recovering lossy derived metadata that is absent from canonical recovery records. + +## Existing Invariant Conflict + +Three existing behaviors conflict: + +1. `verifyRecoveryDestinationRows` requires zero `indexed_files` rows in a recovery destination. +2. `Database.Validate` requires every `search_items.source_path` to exist in `indexed_files`. +3. `BackfillDerived` uses absence from `indexed_files` to identify stored rows whose source is no longer available through normal sync. + +The fix must account for recovered paths for validation without making them look synchronized or excluding them from backfill. + +## Design + +### Reserved accounting marker + +Storage will define one internal reserved value: + +```text +backscroll:recovered +``` + +The value is deliberately outside the lowercase 64-character hexadecimal format produced by SHA-256. It means: + +> Records for this source path were reconstructed from a database recovery plan; no current content hash for the original source is known. + +The marker is provisional. When `SyncFiles` later processes the corresponding source, its existing `INSERT OR REPLACE` into `indexed_files` replaces the marker with the real hash. + +### Recovery destination creation + +`CreateRecoveryDestination` will continue inserting every canonical recovery record into `search_items` inside the existing transaction. Before destination verification and commit, it will also: + +1. Collect each distinct `SourcePath` from the applicable recovery plan. +2. Insert exactly one `indexed_files` row per distinct path. +3. Store `backscroll:recovered` as the row's `hash`. +4. Store `NULL` for `last_indexed`, because recovery did not index the original source at that time. + +The table's primary key on `path` provides uniqueness. No source file is read or written during this operation. + +### Recovery destination verification + +`verifyRecoveryDestinationRows` will replace its current `indexed_files count = 0` rule with exact provisional-accounting verification: + +- the destination has one accounting row per distinct planned `SourcePath`; +- every expected path exists; +- every expected row contains `backscroll:recovered`; +- no additional `indexed_files` rows exist. + +Any missing path, unexpected path, duplicate-equivalent accounting mismatch, or incorrect marker fails destination verification before installation. Existing canonical-record and FTS verification remains unchanged. + +### Public validation + +`Database.Validate` retains its current orphan invariant: every `search_items.source_path` must have a corresponding `indexed_files.path`. + +Recovered destinations now satisfy that invariant through explicit provisional accounting. A row inserted without either normal or recovery accounting remains a genuine orphan and continues to fail validation. + +### Incremental sync + +`GetFileHashes` supplies the skip map used by autosync. It will omit rows whose hash is `backscroll:recovered`. + +Consequences: + +- If an original source still exists, discovery and hashing proceed normally; because the path is absent from the returned skip map, the reader parses it and `SyncFiles` replaces the marker with its real SHA-256. +- If an original source no longer exists, discovery never returns it; the recovered database rows remain perennial and untouched. + +No special branch is required in `SyncFiles`: its existing `INSERT OR REPLACE` behavior performs the state transition atomically with the normal sync transaction. + +### Status accounting + +`GetStats` will exclude `backscroll:recovered` rows from `TotalFiles` and from the `IndexedAt` maximum. Those fields describe sources actually indexed from disk, whereas recovered database records are reported by the existing message counts. This prevents recovery time or provisional paths from masquerading as a successful source sync. + +### Derived-data backfill + +`BackfillDerived` will classify a source path as recovery/expired input when either: + +- no matching `indexed_files` row exists, or +- the matching row's hash is `backscroll:recovered`. + +All existing missing-derivation predicates remain in effect. This preserves idempotency and prevents repeated work after templates, correction signals, and lossy tool events have been derived. + +### Purge lifecycle + +`Purge` already removes an `indexed_files` entry when no `search_items` remain for its path. The same behavior applies to provisional recovery markers, so no marker remains after the last recovered row for a path is purged. + +## Data Flow + +```text +active DB + stranded DB + | + v + canonical recovery plan + | + +--> search_items: canonical recovered rows + | + +--> indexed_files: one path -> backscroll:recovered marker + | + v + destination verification -> atomic installation -> validate succeeds + +Later autosync: + source absent -> recovered rows and marker remain; backfill may derive data + source present -> GetFileHashes omits marker -> parse -> SyncFiles writes real SHA-256 +``` + +## Error Handling + +- Marker insertion failures abort and roll back destination creation. +- Accounting verification failures reject the destination before installation. +- The independent post-commit immutable verification applies the same accounting rules. +- Errors continue using the existing recovery destination error wrappers and cleanup behavior. +- Autosync and backfill database errors keep their existing contextual wrapping. +- No fallback invents or accepts a normal-looking content hash. + +## Testing + +### CLI regression + +Add a full command-level regression using supported active and stranded databases: + +1. Capture the active database bytes. +2. Run `recover --from `. +3. Run `validate --indexed-only` against the installed destination and require success. +4. Confirm the reported backup is byte-identical to the original active database. +5. Confirm all expected recovered records remain queryable. + +### Storage tests + +Cover these invariants directly: + +- Recovery creates one marked accounting row per distinct planned source path. +- Destination verification rejects a missing accounting row. +- Destination verification rejects an incorrect marker. +- Destination verification rejects an unexpected accounting path. +- Messages and tool records remain queryable through their respective FTS indexes. +- `GetFileHashes` returns real hashes but omits provisional recovery markers. +- `GetStats` excludes provisional markers from file count and last-indexed time. +- `BackfillDerived` processes a marked recovered path. +- `SyncFiles` replaces a marker with the real source hash. +- `Purge` removes a marker after deleting the final row for its path. +- The existing genuine-orphan validation regression continues to fail as expected. + +### Verification commands + +```bash +just check +just test +just ci +``` + +## Documentation Impact + +This change adds no CLI option and no migration. Internal comments should document the marker contract where it is defined and where queries intentionally include or exclude it. The repository architecture documentation does not need a new package entry because no package is added or removed. diff --git a/go.mod b/go.mod index b43e82d..a2acf08 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/pablontiv/picokit v0.5.1 github.com/pelletier/go-toml/v2 v2.3.1 github.com/spf13/cobra v1.10.2 + golang.org/x/sys v0.42.0 modernc.org/sqlite v1.50.1 ) @@ -17,7 +18,6 @@ require ( github.com/ncruces/go-strftime v1.0.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/spf13/pflag v1.0.9 // indirect - golang.org/x/sys v0.42.0 // indirect modernc.org/libc v1.72.3 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect diff --git a/internal/compat/catalog.go b/internal/compat/catalog.go new file mode 100644 index 0000000..79c9117 --- /dev/null +++ b/internal/compat/catalog.go @@ -0,0 +1,205 @@ +package compat + +import ( + "crypto/sha256" + "embed" + "encoding/json" + "fmt" + "io/fs" + "strings" +) + +//go:embed testdata/release-schemas/* +var embeddedReleaseSchemaFS embed.FS + +var releaseSchemaFS fs.FS = embeddedReleaseSchemaFS + +type Catalog struct { + FirstGoRelease string + LatestGoRelease string + Releases []catalogRelease + UnmanifestedFixtures []catalogFixture + + lineages map[string]Lineage + currentSignature string +} + +type catalogRelease struct { + Tag string + Fixture string + ProvenanceSHA256 string + Signature string + AppliedVersion int + HasSourceMetadata bool +} + +type catalogFixture struct { + Fixture string + ProvenanceSHA256 string + Signature string + AppliedVersion int + HasSourceMetadata bool + Provenance string +} + +type Lineage struct { + shape SchemaShape + remainingSteps []MigrationStep +} + +func (c Catalog) BySignature(signature string) (Lineage, bool) { + lineage, ok := c.lineages[signature] + return lineage, ok +} + +func (c Catalog) CurrentSignature() string { + return c.currentSignature +} + +func (l Lineage) RemainingSteps() []MigrationStep { + steps := make([]MigrationStep, len(l.remainingSteps)) + copy(steps, l.remainingSteps) + return steps +} + +func LoadCatalog() (Catalog, error) { + catalog, err := loadCatalogFromFS(releaseSchemaFS) + if err != nil { + return Catalog{}, err + } + if err := catalog.attachLineages(); err != nil { + return Catalog{}, err + } + return catalog, nil +} + +func loadCatalogFromFS(fsys fs.FS) (Catalog, error) { + data, err := fs.ReadFile(fsys, "testdata/release-schemas/manifest.json") + if err != nil { + return Catalog{}, fmt.Errorf("read release schema catalog: %w", err) + } + + var catalog Catalog + if err := json.Unmarshal(data, &catalog); err != nil { + return Catalog{}, fmt.Errorf("parse release schema catalog: %w", err) + } + if catalog.FirstGoRelease != "v0.3.7" { + return Catalog{}, fmt.Errorf("release schema catalog first Go release = %q, want v0.3.7", catalog.FirstGoRelease) + } + if compareSemver(catalog.LatestGoRelease, "v3.2.5") < 0 { + return Catalog{}, fmt.Errorf("release schema catalog latest Go release = %q, want at least v3.2.5", catalog.LatestGoRelease) + } + + seen := make(map[string]bool, len(catalog.Releases)) + for _, release := range catalog.Releases { + if release.Tag == "" { + return Catalog{}, fmt.Errorf("release schema catalog has release without tag: %+v", release) + } + if seen[release.Tag] { + return Catalog{}, fmt.Errorf("release schema catalog has duplicate release tag %q", release.Tag) + } + seen[release.Tag] = true + if err := validateCatalogFixture(fsys, fixtureFromRelease(release)); err != nil { + return Catalog{}, err + } + } + for _, fixture := range catalog.UnmanifestedFixtures { + if fixture.Provenance == "" { + return Catalog{}, fmt.Errorf("unmanifested release schema fixture %q lacks provenance note", fixture.Fixture) + } + if err := validateCatalogFixture(fsys, fixture); err != nil { + return Catalog{}, err + } + } + if !seen[catalog.FirstGoRelease] || !seen[catalog.LatestGoRelease] { + return Catalog{}, fmt.Errorf("release schema catalog endpoints missing: %s..%s", catalog.FirstGoRelease, catalog.LatestGoRelease) + } + + return catalog, nil +} + +func fixtureFromRelease(release catalogRelease) catalogFixture { + return catalogFixture{ + Fixture: release.Fixture, + ProvenanceSHA256: release.ProvenanceSHA256, + Signature: release.Signature, + AppliedVersion: release.AppliedVersion, + HasSourceMetadata: release.HasSourceMetadata, + } +} + +func validateCatalogFixture(fsys fs.FS, fixture catalogFixture) error { + if fixture.Fixture == "" || fixture.ProvenanceSHA256 == "" || fixture.Signature == "" || fixture.AppliedVersion == 0 { + return fmt.Errorf("release schema catalog has incomplete fixture mapping: %+v", fixture) + } + if !strings.HasPrefix(fixture.Signature, "sha256:") { + return fmt.Errorf("release schema fixture %q signature = %q, want sha256", fixture.Fixture, fixture.Signature) + } + fixturePath := "testdata/release-schemas/" + fixture.Fixture + fixtureBytes, err := fs.ReadFile(fsys, fixturePath) + if err != nil { + return fmt.Errorf("release schema fixture %q: %w", fixture.Fixture, err) + } + actualSHA256 := fmt.Sprintf("%x", sha256.Sum256(fixtureBytes)) + if actualSHA256 != fixture.ProvenanceSHA256 { + return fmt.Errorf("release schema fixture %q SHA-256 = %s, want %s", fixture.Fixture, actualSHA256, fixture.ProvenanceSHA256) + } + return nil +} + +func (c Catalog) schemaFixtures() []catalogFixture { + seen := map[string]bool{} + fixtures := make([]catalogFixture, 0, len(c.Releases)+len(c.UnmanifestedFixtures)) + for _, release := range c.Releases { + fixture := fixtureFromRelease(release) + if seen[fixture.Fixture] { + continue + } + seen[fixture.Fixture] = true + fixtures = append(fixtures, fixture) + } + for _, fixture := range c.UnmanifestedFixtures { + if seen[fixture.Fixture] { + continue + } + seen[fixture.Fixture] = true + fixtures = append(fixtures, fixture) + } + return fixtures +} + +func (c *Catalog) attachLineages() error { + lineages := map[string]Lineage{} + for _, fixture := range c.schemaFixtures() { + shape := SchemaShape{AppliedVersion: fixture.AppliedVersion, Signature: fixture.Signature} + lineages[fixture.Signature] = Lineage{ + shape: shape, + remainingSteps: remainingStepsFor(fixture.AppliedVersion, fixture.HasSourceMetadata), + } + } + for _, release := range c.Releases { + if release.Tag == c.LatestGoRelease { + c.currentSignature = release.Signature + break + } + } + if c.currentSignature == "" { + return fmt.Errorf("release schema catalog latest release %q has no signature", c.LatestGoRelease) + } + c.lineages = lineages + return nil +} + +func compareSemver(left, right string) int { + var lMajor, lMinor, lPatch int + var rMajor, rMinor, rPatch int + _, _ = fmt.Sscanf(left, "v%d.%d.%d", &lMajor, &lMinor, &lPatch) + _, _ = fmt.Sscanf(right, "v%d.%d.%d", &rMajor, &rMinor, &rPatch) + if lMajor != rMajor { + return lMajor - rMajor + } + if lMinor != rMinor { + return lMinor - rMinor + } + return lPatch - rPatch +} diff --git a/internal/compat/catalog_test.go b/internal/compat/catalog_test.go new file mode 100644 index 0000000..e5bd892 --- /dev/null +++ b/internal/compat/catalog_test.go @@ -0,0 +1,305 @@ +package compat + +import ( + "context" + "crypto/sha256" + "database/sql" + "fmt" + "io/fs" + "path/filepath" + "reflect" + "sort" + "strings" + "testing" + "testing/fstest" +) + +func TestCheckedInReleaseSchemaManifestIsComplete(t *testing.T) { + catalog, err := LoadCatalog() + if err != nil { + t.Fatal(err) + } + if catalog.FirstGoRelease != "v0.3.7" || catalog.LatestGoRelease != "v3.2.5" { + t.Fatalf("catalog bounds = %s..%s", catalog.FirstGoRelease, catalog.LatestGoRelease) + } + seen := map[string]bool{} + for _, release := range catalog.Releases { + if release.Tag == "" || release.Fixture == "" || release.ProvenanceSHA256 == "" || seen[release.Tag] { + t.Fatalf("invalid release mapping: %+v", release) + } + seen[release.Tag] = true + if _, err := fs.Stat(releaseSchemaFS, "testdata/release-schemas/"+release.Fixture); err != nil { + t.Fatalf("fixture %q: %v", release.Fixture, err) + } + } + if !seen["v0.3.7"] || !seen["v3.2.5"] { + t.Fatalf("release endpoints missing: %v", seen) + } +} + +func TestReleaseSchemaManifestRejectsMissingFixture(t *testing.T) { + _, err := loadCatalogFromFS(fstest.MapFS{ + "testdata/release-schemas/manifest.json": {Data: []byte(`{ + "FirstGoRelease": "v0.3.7", + "LatestGoRelease": "v3.2.5", + "Releases": [{"Tag": "v0.3.7", "Fixture": "missing.sql", "ProvenanceSHA256": "abc123"}] + }`)}, + }) + if err == nil || !strings.Contains(err.Error(), "missing.sql") { + t.Fatalf("missing fixture error = %v", err) + } +} + +func TestReleaseSchemaManifestRejectsLatestBeforeV3_2_5(t *testing.T) { + _, err := loadCatalogFromFS(fstest.MapFS{ + "testdata/release-schemas/manifest.json": {Data: []byte(`{ + "FirstGoRelease": "v0.3.7", + "LatestGoRelease": "v3.2.4", + "Releases": [{"Tag": "v0.3.7", "Fixture": "v1.sql", "ProvenanceSHA256": "abc123"}] + }`)}, + "testdata/release-schemas/v1.sql": {Data: []byte("-- fixture\n")}, + }) + if err == nil || !strings.Contains(err.Error(), "v3.2.5") { + t.Fatalf("latest bound error = %v", err) + } +} + +func TestLoadCatalogUsesCheckedInSignaturesWithoutExecutingFixtureSQL(t *testing.T) { + poisonSQL := []byte("BEGIN TRANSACTION; this is intentionally not executable fixture SQL; COMMIT;") + poisonSHA := fmt.Sprintf("%x", sha256.Sum256(poisonSQL)) + + withReleaseSchemaFS(t, fstest.MapFS{ + "testdata/release-schemas/manifest.json": {Data: []byte(fmt.Sprintf(`{ + "FirstGoRelease": "v0.3.7", + "LatestGoRelease": "v3.2.5", + "Releases": [ + {"Tag": "v0.3.7", "Fixture": "poison.sql", "ProvenanceSHA256": %q, "Signature": "sha256:poisoned", "AppliedVersion": 13}, + {"Tag": "v3.2.5", "Fixture": "poison.sql", "ProvenanceSHA256": %q, "Signature": "sha256:poisoned", "AppliedVersion": 13} + ] + }`, poisonSHA, poisonSHA))}, + "testdata/release-schemas/poison.sql": {Data: poisonSQL}, + }) + + catalog, err := LoadCatalog() + if err != nil { + t.Fatalf("load catalog executed fixture SQL or rejected checked-in signature data: %v", err) + } + if got := catalog.CurrentSignature(); got != "sha256:poisoned" { + t.Fatalf("current signature = %q, want checked-in signature", got) + } +} + +func TestCurrentSignatureFollowsLatestReleaseMapping(t *testing.T) { + fixtureSQL := []byte("CREATE TABLE schema_migrations (version INTEGER PRIMARY KEY, name TEXT NOT NULL, applied_on TEXT NOT NULL, checksum TEXT NOT NULL);") + fixtureSHA := fmt.Sprintf("%x", sha256.Sum256(fixtureSQL)) + + withReleaseSchemaFS(t, fstest.MapFS{ + "testdata/release-schemas/manifest.json": {Data: []byte(fmt.Sprintf(`{ + "FirstGoRelease": "v0.3.7", + "LatestGoRelease": "v3.2.6", + "Releases": [ + {"Tag": "v0.3.7", "Fixture": "v13.sql", "ProvenanceSHA256": %q, "Signature": "sha256:old-latest", "AppliedVersion": 13}, + {"Tag": "v3.2.5", "Fixture": "v13.sql", "ProvenanceSHA256": %q, "Signature": "sha256:old-latest", "AppliedVersion": 13}, + {"Tag": "v3.2.6", "Fixture": "v14.sql", "ProvenanceSHA256": %q, "Signature": "sha256:new-latest", "AppliedVersion": 14} + ] + }`, fixtureSHA, fixtureSHA, fixtureSHA))}, + "testdata/release-schemas/v13.sql": {Data: fixtureSQL}, + "testdata/release-schemas/v14.sql": {Data: fixtureSQL}, + }) + + catalog, err := LoadCatalog() + if err != nil { + t.Fatal(err) + } + if got := catalog.CurrentSignature(); got != "sha256:new-latest" { + t.Fatalf("current signature = %q, want latest release mapping signature", got) + } +} + +func TestReleaseSchemaFixtureSignaturesMatchCheckedInSQL(t *testing.T) { + catalog, err := LoadCatalog() + if err != nil { + t.Fatal(err) + } + checked := map[string]bool{} + for _, entry := range catalogShapeFixtures(t, catalog) { + t.Run(entry.fixture, func(t *testing.T) { + if checked[entry.fixture] { + return + } + checked[entry.fixture] = true + db := openFixtureCopy(t, entry.fixture) + defer db.Close() + + shape, err := inspectShape(context.Background(), db) + if err != nil { + t.Fatal(err) + } + if shape.Signature != entry.signature { + t.Fatalf("signature = %s, want checked-in %s", shape.Signature, entry.signature) + } + }) + } +} + +func TestReleaseSchemaManifestRejectsFixtureHashDrift(t *testing.T) { + original, err := fs.ReadFile(releaseSchemaFS, "testdata/release-schemas/v1.sql") + if err != nil { + t.Fatal(err) + } + provenance := fmt.Sprintf("%x", sha256.Sum256(original)) + modified := append([]byte(nil), original...) + modified = append(modified, []byte("\n-- modified fixture bytes\n")...) + + _, err = loadCatalogFromFS(fstest.MapFS{ + "testdata/release-schemas/manifest.json": {Data: []byte(fmt.Sprintf(`{ + "FirstGoRelease": "v0.3.7", + "LatestGoRelease": "v3.2.5", + "Releases": [ + {"Tag": "v0.3.7", "Fixture": "v1.sql", "ProvenanceSHA256": %q, "Signature": "sha256:fixture", "AppliedVersion": 1, "HasSourceMetadata": true}, + {"Tag": "v3.2.5", "Fixture": "v1.sql", "ProvenanceSHA256": %q, "Signature": "sha256:fixture", "AppliedVersion": 1, "HasSourceMetadata": true} + ] + }`, provenance, provenance))}, + "testdata/release-schemas/v1.sql": {Data: modified}, + }) + if err == nil || !strings.Contains(err.Error(), "SHA-256") { + t.Fatalf("fixture hash drift error = %v", err) + } +} + +func TestFixtureMigrationRowsMatchPublishedCurrentLedger(t *testing.T) { + authoritative := loadPublishedCurrentMigrationRows(t) + + fixturePaths, err := fs.Glob(releaseSchemaFS, "testdata/release-schemas/*.sql") + if err != nil { + t.Fatal(err) + } + sort.Strings(fixturePaths) + if len(fixturePaths) == 0 { + t.Fatal("no SQL fixtures found") + } + + for _, fixturePath := range fixturePaths { + t.Run(filepath.Base(fixturePath), func(t *testing.T) { + fixtureSQL, err := fs.ReadFile(releaseSchemaFS, fixturePath) + if err != nil { + t.Fatal(err) + } + fixtureRows := loadFixtureMigrationRows(t, fixtureSQL) + if len(fixtureRows) == 0 { + t.Fatal("fixture has no schema_migrations rows") + } + for _, row := range fixtureRows { + expected, ok := authoritative[row.version] + if !ok { + t.Fatalf("fixture has unknown migration version %d", row.version) + } + if row.name != expected.name || row.checksum != expected.checksum { + t.Fatalf("migration %d row = (%q, %q), want (%q, %q)", row.version, row.name, row.checksum, expected.name, expected.checksum) + } + } + }) + } +} + +func TestUnmanifestedFixtureShapesAreDocumentedLocally(t *testing.T) { + for _, fixture := range []string{"v2.sql", "v3-no-source-metadata.sql", "v5-without-source-metadata.sql", "v6.sql"} { + t.Run(fixture, func(t *testing.T) { + data, err := fs.ReadFile(releaseSchemaFS, "testdata/release-schemas/"+fixture) + if err != nil { + t.Fatal(err) + } + text := string(data) + if !strings.Contains(text, "No manifest release tag maps to this fixture.") || !strings.Contains(text, "compatibility triangulation") { + t.Fatalf("fixture lacks local documentation for its unmanifested shape") + } + }) + } +} + +type migrationRow struct { + version int + name string + checksum string +} + +func catalogShapeFixtures(t *testing.T, catalog Catalog) []struct{ fixture, signature string } { + t.Helper() + var result []struct{ fixture, signature string } + catalogValue := reflect.ValueOf(catalog) + for _, fieldName := range []string{"Releases", "UnmanifestedFixtures"} { + field := catalogValue.FieldByName(fieldName) + if !field.IsValid() { + continue + } + for i := 0; i < field.Len(); i++ { + entry := field.Index(i) + fixture := entry.FieldByName("Fixture") + signature := entry.FieldByName("Signature") + if !fixture.IsValid() || !signature.IsValid() || fixture.String() == "" || signature.String() == "" { + t.Fatalf("catalog %s[%d] lacks fixture signature data", fieldName, i) + } + result = append(result, struct{ fixture, signature string }{fixture: fixture.String(), signature: signature.String()}) + } + } + if len(result) == 0 { + t.Fatal("catalog has no fixture signature data") + } + return result +} + +func withReleaseSchemaFS(t *testing.T, fsys fs.FS) { + t.Helper() + original := releaseSchemaFS + releaseSchemaFS = fsys + t.Cleanup(func() { releaseSchemaFS = original }) +} + +func loadPublishedCurrentMigrationRows(t *testing.T) map[int]migrationRow { + t.Helper() + + fixtureSQL, err := fs.ReadFile(releaseSchemaFS, "testdata/release-schemas/v13.sql") + if err != nil { + t.Fatal(err) + } + rows := loadFixtureMigrationRows(t, fixtureSQL) + + result := map[int]migrationRow{} + for _, row := range rows { + result[row.version] = row + } + return result +} + +func loadFixtureMigrationRows(t *testing.T, fixtureSQL []byte) []migrationRow { + t.Helper() + + db, err := sql.Open("sqlite", ":memory:") + if err != nil { + t.Fatal(err) + } + defer db.Close() + + if _, err := db.Exec(string(fixtureSQL)); err != nil { + t.Fatal(err) + } + + rows, err := db.Query("SELECT version, name, checksum FROM schema_migrations ORDER BY version") + if err != nil { + t.Fatal(err) + } + defer rows.Close() + + var result []migrationRow + for rows.Next() { + var row migrationRow + if err := rows.Scan(&row.version, &row.name, &row.checksum); err != nil { + t.Fatal(err) + } + result = append(result, row) + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + return result +} diff --git a/internal/compat/recovery.go b/internal/compat/recovery.go new file mode 100644 index 0000000..19562b6 --- /dev/null +++ b/internal/compat/recovery.go @@ -0,0 +1,234 @@ +package compat + +import ( + "bytes" + "crypto/sha256" + "encoding/binary" + "errors" + "fmt" + "sort" + + "github.com/google/uuid" + "github.com/pablontiv/backscroll/internal/models" +) + +var errUnsafeIdentity = errors.New("unsafe recovery identity") + +type recordIdentity struct { + Kind string + UUID string + SourcePath string + Ordinal int64 +} + +// PlanRecovery builds a deterministic, read-only union of canonical recovery +// records. If any input row cannot be interpreted safely, or if one identity has +// multiple canonical payloads, the returned records are intentionally empty and +// diagnostics describe every rejected occurrence. +func PlanRecovery(inputs []RecoveryInput) (RecoveryPlan, []Diagnostic, error) { + plan := RecoveryPlan{InputShapes: make([]SchemaShape, 0, len(inputs))} + byIdentity := map[recordIdentity]map[string][]recordOccurrence{} + var diagnostics []Diagnostic + + for inputIndex, input := range inputs { + plan.InputShapes = append(plan.InputShapes, input.Shape) + for rowIndex, record := range input.Records { + identity, err := identityOf(record) + if err != nil { + diagnostics = append(diagnostics, uninterpretablePlanDiagnostic(inputIndex, rowIndex, record, err)) + continue + } + hash := payloadHash(record, identity) + if byIdentity[identity] == nil { + byIdentity[identity] = map[string][]recordOccurrence{} + } + byIdentity[identity][hash] = append(byIdentity[identity][hash], recordOccurrence{ + inputIndex: inputIndex, + rowIndex: rowIndex, + identity: identity, + hash: hash, + record: record, + }) + } + } + + identities := make([]recordIdentity, 0, len(byIdentity)) + for identity := range byIdentity { + identities = append(identities, identity) + } + sort.Slice(identities, func(i, j int) bool { return identityLess(identities[i], identities[j]) }) + + var canonical []plannedRecord + for _, identity := range identities { + groups := byIdentity[identity] + if len(groups) > 1 { + var rejected []recordOccurrence + for _, occurrences := range groups { + rejected = append(rejected, occurrences...) + } + sortOccurrences(rejected) + for _, occurrence := range rejected { + diagnostics = append(diagnostics, conflictPlanDiagnostic(occurrence, len(groups))) + } + continue + } + for hash, occurrences := range groups { + sortOccurrences(occurrences) + canonical = append(canonical, plannedRecord{ + identity: identity, + hash: hash, + record: occurrences[0].record, + }) + plan.ExactDuplicates += len(occurrences) - 1 + } + } + + if len(diagnostics) != 0 { + plan.Records = nil + plan.ExactDuplicates = 0 + return plan, diagnostics, nil + } + + sort.Slice(canonical, func(i, j int) bool { + if !sameIdentity(canonical[i].identity, canonical[j].identity) { + return identityLess(canonical[i].identity, canonical[j].identity) + } + return canonical[i].hash < canonical[j].hash + }) + plan.Records = make([]CanonicalRecord, 0, len(canonical)) + for _, record := range canonical { + plan.Records = append(plan.Records, CanonicalRecord{ + Record: record.record, + PayloadHash: record.hash, + }) + } + return plan, nil, nil +} + +func identityOf(r models.IndexedRecord) (recordIdentity, error) { + if r.UUID != nil && *r.UUID != "" { + if _, err := uuid.Parse(*r.UUID); err != nil { + return recordIdentity{}, err + } + return recordIdentity{Kind: "uuid", UUID: *r.UUID}, nil + } + if r.SourcePath == "" || r.Ordinal < 0 { + return recordIdentity{}, errUnsafeIdentity + } + return recordIdentity{Kind: "path_ordinal", SourcePath: r.SourcePath, Ordinal: r.Ordinal}, nil +} + +type recordOccurrence struct { + inputIndex int + rowIndex int + identity recordIdentity + hash string + record models.IndexedRecord +} + +type plannedRecord struct { + identity recordIdentity + hash string + record models.IndexedRecord +} + +func payloadHash(record models.IndexedRecord, identity recordIdentity) string { + var encoded bytes.Buffer + writeString(&encoded, record.Source) + if identity.Kind != "path_ordinal" { + writeString(&encoded, record.SourcePath) + writeInt64(&encoded, record.Ordinal) + } + writeString(&encoded, record.Role) + writeString(&encoded, record.Text) + writeStringPtr(&encoded, record.Project) + if identity.Kind == "path_ordinal" { + writeString(&encoded, "") + } + writeStringPtr(&encoded, record.Timestamp) + writeString(&encoded, record.ContentType) + + sum := sha256.Sum256(encoded.Bytes()) + return fmt.Sprintf("%x", sum) +} + +func writeString(buffer *bytes.Buffer, value string) { + _ = buffer.WriteByte(1) + writeLengthPrefixed(buffer, []byte(value)) +} + +func writeStringPtr(buffer *bytes.Buffer, value *string) { + if value == nil { + _ = buffer.WriteByte(0) + return + } + _ = buffer.WriteByte(1) + writeLengthPrefixed(buffer, []byte(*value)) +} + +func writeInt64(buffer *bytes.Buffer, value int64) { + _ = buffer.WriteByte(1) + writeLengthPrefixed(buffer, []byte(fmt.Sprintf("%d", value))) +} + +func writeLengthPrefixed(buffer *bytes.Buffer, value []byte) { + var length [8]byte + binary.BigEndian.PutUint64(length[:], uint64(len(value))) + _, _ = buffer.Write(length[:]) + _, _ = buffer.Write(value) +} + +func identityLess(left, right recordIdentity) bool { + if left.Kind != right.Kind { + return left.Kind < right.Kind + } + if left.UUID != right.UUID { + return left.UUID < right.UUID + } + if left.SourcePath != right.SourcePath { + return left.SourcePath < right.SourcePath + } + return left.Ordinal < right.Ordinal +} + +func sameIdentity(left, right recordIdentity) bool { + return left.Kind == right.Kind && left.UUID == right.UUID && left.SourcePath == right.SourcePath && left.Ordinal == right.Ordinal +} + +func sortOccurrences(occurrences []recordOccurrence) { + sort.Slice(occurrences, func(i, j int) bool { + if occurrences[i].inputIndex != occurrences[j].inputIndex { + return occurrences[i].inputIndex < occurrences[j].inputIndex + } + return occurrences[i].rowIndex < occurrences[j].rowIndex + }) +} + +func uninterpretablePlanDiagnostic(inputIndex, rowIndex int, record models.IndexedRecord, cause error) Diagnostic { + identityEvidence := fmt.Sprintf("uuid=%s source_path=%q ordinal=%d", pointerEvidence(record.UUID), record.SourcePath, record.Ordinal) + return Diagnostic{ + Code: CodeUninterpretableRow, + Summary: fmt.Sprintf("recovery input %d row %d has uninterpretable identity (%s): %v", inputIndex, rowIndex, identityEvidence, cause), + } +} + +func conflictPlanDiagnostic(occurrence recordOccurrence, payloadCount int) Diagnostic { + return Diagnostic{ + Code: CodeRecoveryConflict, + Summary: fmt.Sprintf("recovery input %d row %d conflicts for %s with %d payload hashes", occurrence.inputIndex, occurrence.rowIndex, describeIdentity(occurrence.identity), payloadCount), + } +} + +func describeIdentity(identity recordIdentity) string { + if identity.Kind == "uuid" { + return "uuid " + identity.UUID + } + return fmt.Sprintf("path_ordinal source_path=%q ordinal=%d", identity.SourcePath, identity.Ordinal) +} + +func pointerEvidence(value *string) string { + if value == nil { + return "" + } + return fmt.Sprintf("%q", *value) +} diff --git a/internal/compat/recovery_test.go b/internal/compat/recovery_test.go new file mode 100644 index 0000000..55eda14 --- /dev/null +++ b/internal/compat/recovery_test.go @@ -0,0 +1,378 @@ +package compat + +import ( + "fmt" + "reflect" + "strings" + "testing" + + "github.com/pablontiv/backscroll/internal/models" +) + +const ( + uuidA = "11111111-1111-4111-8111-111111111111" + uuidB = "22222222-2222-4222-8222-222222222222" + uuidC = "33333333-3333-4333-8333-333333333333" +) + +func TestRecoverIdentityAndConflictMatrixAcrossInputs(t *testing.T) { + empty := "" + emptyProject := "" + + tests := []struct { + name string + records []models.IndexedRecord + wantRecords int + wantDuplicates int + wantDiagnostics map[Code]int + wantSummaryParts []string + }{ + { + name: "valid UUID identity beats differing path ordinal and reports payload conflict", + records: []models.IndexedRecord{ + recordWithUUID(uuidA, "/active/path", 1), + recordWithUUID(uuidA, "/stranded/path", 9), + }, + wantDiagnostics: map[Code]int{CodeRecoveryConflict: 2}, + wantSummaryParts: []string{uuidA}, + }, + { + name: "empty UUID falls back to exact path ordinal and collapses nil and empty UUID", + records: []models.IndexedRecord{ + recordWithoutUUID("/exact/path", 7, nil), + recordWithoutUUID("/exact/path", 7, &empty), + }, + wantRecords: 1, + wantDuplicates: 1, + }, + { + name: "equal UUID identity and equal canonical payload collapses", + records: []models.IndexedRecord{ + recordWithUUID(uuidA, "/same/path", 1), + recordWithUUID(uuidA, "/same/path", 1), + }, + wantRecords: 1, + wantDuplicates: 1, + }, + { + name: "equal UUID identity and different text payload conflicts", + records: []models.IndexedRecord{ + recordWithUUID(uuidA, "/same/path", 1), + withText(recordWithUUID(uuidA, "/same/path", 1), "different text"), + }, + wantDiagnostics: map[Code]int{CodeRecoveryConflict: 2}, + wantSummaryParts: []string{uuidA}, + }, + { + name: "equal path ordinal identity and nil versus empty project pointer conflicts", + records: []models.IndexedRecord{ + recordWithoutUUID("/same/path", 1, nil), + withProject(recordWithoutUUID("/same/path", 1, nil), &emptyProject), + }, + wantDiagnostics: map[Code]int{CodeRecoveryConflict: 2}, + wantSummaryParts: []string{"/same/path", "1"}, + }, + { + name: "equal content under different UUID identities remains two records", + records: []models.IndexedRecord{ + recordWithUUID(uuidA, "/same/path", 1), + recordWithUUID(uuidB, "/same/path", 1), + }, + wantRecords: 2, + }, + { + name: "invalid non-empty UUID is uninterpretable and never falls back to path ordinal", + records: []models.IndexedRecord{ + recordWithUUID("not-a-uuid", "/safe/path", 3), + recordWithoutUUID("/safe/path", 3, nil), + }, + wantDiagnostics: map[Code]int{CodeUninterpretableRow: 1}, + wantSummaryParts: []string{"not-a-uuid"}, + }, + { + name: "empty UUID with empty path or negative ordinal is uninterpretable", + records: []models.IndexedRecord{ + recordWithoutUUID("", 3, nil), + recordWithoutUUID("/safe/path", -1, &empty), + }, + wantDiagnostics: map[Code]int{CodeUninterpretableRow: 2}, + wantSummaryParts: []string{"/safe/path", "-1"}, + }, + } + + for _, tt := range tests { + for _, layout := range []struct { + name string + inputs []RecoveryInput + }{ + {name: "within-active-input", inputs: []RecoveryInput{recoveryInput("active", 13, tt.records...)}}, + {name: "across-active-and-stranded-inputs", inputs: splitAcrossActiveAndStranded(tt.records)}, + } { + t.Run(tt.name+"/"+layout.name, func(t *testing.T) { + plan, diagnostics, err := PlanRecovery(layout.inputs) + if err != nil { + t.Fatalf("PlanRecovery error = %v", err) + } + + assertInputShapes(t, plan, layout.inputs) + assertDiagnosticCounts(t, diagnostics, tt.wantDiagnostics) + assertDiagnosticSummariesContain(t, diagnostics, tt.wantSummaryParts) + + if len(tt.wantDiagnostics) > 0 { + if len(plan.Records) != 0 { + t.Fatalf("plan records = %d, want 0 when diagnostics reject the plan", len(plan.Records)) + } + return + } + if len(diagnostics) != 0 { + t.Fatalf("diagnostics = %+v, want none", diagnostics) + } + if len(plan.Records) != tt.wantRecords || plan.ExactDuplicates != tt.wantDuplicates { + t.Fatalf("plan records=%d duplicates=%d, want records=%d duplicates=%d", len(plan.Records), plan.ExactDuplicates, tt.wantRecords, tt.wantDuplicates) + } + inputRows := 0 + for _, input := range layout.inputs { + inputRows += len(input.Records) + } + if got := len(plan.Records) + plan.ExactDuplicates; got != inputRows { + t.Fatalf("clean accounting records + duplicates = %d, want input rows %d", got, inputRows) + } + }) + } + } +} + +func TestPlanRecoveryAccountsForEveryInputRow(t *testing.T) { + active := recoveryInput("active", 13, + recordWithUUID(uuidA, "/active/a", 1), + recordWithUUID(uuidA, "/active/a", 1), + recordWithoutUUID("/active/path", 2, nil), + ) + stranded := recoveryInput("stranded", 7, + recordWithoutUUID("/active/path", 2, nil), + recordWithUUID(uuidB, "/stranded/b", 3), + ) + + plan, diagnostics, err := PlanRecovery([]RecoveryInput{active, stranded}) + if err != nil || len(diagnostics) != 0 { + t.Fatalf("PlanRecovery error=%v diagnostics=%+v", err, diagnostics) + } + assertInputShapes(t, plan, []RecoveryInput{active, stranded}) + if len(plan.Records) != 3 || plan.ExactDuplicates != 2 { + t.Fatalf("records=%d duplicates=%d, want records=3 duplicates=2", len(plan.Records), plan.ExactDuplicates) + } + if got, want := len(plan.Records)+plan.ExactDuplicates, len(active.Records)+len(stranded.Records); got != want { + t.Fatalf("records + duplicates = %d, want input rows %d", got, want) + } + + conflicting := recoveryInput("conflicting", 6, + withText(recordWithUUID(uuidC, "/conflict", 1), "left"), + withText(recordWithUUID(uuidC, "/conflict", 1), "right"), + recordWithUUID("not-a-uuid", "/safe", 0), + ) + plan, diagnostics, err = PlanRecovery([]RecoveryInput{active, conflicting}) + if err != nil { + t.Fatalf("PlanRecovery conflict error = %v", err) + } + if len(plan.Records) != 0 { + t.Fatalf("conflict plan records = %d, want 0", len(plan.Records)) + } + assertDiagnosticCounts(t, diagnostics, map[Code]int{CodeRecoveryConflict: 2, CodeUninterpretableRow: 1}) + assertDiagnosticSummariesContain(t, diagnostics, []string{uuidC, "not-a-uuid"}) +} + +func TestPlanRecoveryPreservesDistinctHashEquivalentIdentities(t *testing.T) { + first := recordWithUUID(uuidA, "/same/path", 1) + second := recordWithUUID(uuidB, "/same/path", 1) + ambiguousA := withSourceAndRole(recordWithUUID(uuidC, "/injective", 1), "ab", "c") + ambiguousB := withSourceAndRole(recordWithUUID("44444444-4444-4444-8444-444444444444", "/injective", 1), "a", "bc") + + plan, diagnostics, err := PlanRecovery([]RecoveryInput{recoveryInput("active", 13, first, second, ambiguousA, ambiguousB)}) + if err != nil || len(diagnostics) != 0 { + t.Fatalf("PlanRecovery error=%v diagnostics=%+v", err, diagnostics) + } + if len(plan.Records) != 4 { + t.Fatalf("records = %d, want 4 distinct identities", len(plan.Records)) + } + + hashesByUUID := map[string]string{} + for _, record := range plan.Records { + if record.Record.UUID == nil { + t.Fatalf("record without UUID in UUID-only test: %+v", record.Record) + } + hashesByUUID[*record.Record.UUID] = record.PayloadHash + } + if hashesByUUID[uuidA] == "" || hashesByUUID[uuidA] != hashesByUUID[uuidB] { + t.Fatalf("equal payload under distinct identities hashes = %q and %q, want equal non-empty hashes", hashesByUUID[uuidA], hashesByUUID[uuidB]) + } + if hashesByUUID[uuidC] == "" || hashesByUUID[uuidC] == hashesByUUID["44444444-4444-4444-8444-444444444444"] { + t.Fatalf("ambiguous field payload hashes = %q and %q, want distinct length-prefixed hashes", hashesByUUID[uuidC], hashesByUUID["44444444-4444-4444-8444-444444444444"]) + } +} + +func TestPlanRecoveryIsDeterministic(t *testing.T) { + inputs := []RecoveryInput{ + recoveryInput("active", 13, + recordWithUUID(uuidB, "/uuid/b", 2), + recordWithoutUUID("/path/z", 9, nil), + recordWithUUID(uuidA, "/uuid/a", 1), + recordWithoutUUID("/path/a", 1, nil), + ), + recoveryInput("stranded", 7, + recordWithUUID(uuidC, "/uuid/c", 3), + ), + } + reordered := []RecoveryInput{ + recoveryInput("active", 13, + recordWithoutUUID("/path/a", 1, nil), + recordWithUUID(uuidA, "/uuid/a", 1), + recordWithoutUUID("/path/z", 9, nil), + recordWithUUID(uuidB, "/uuid/b", 2), + ), + recoveryInput("stranded", 7, + recordWithUUID(uuidC, "/uuid/c", 3), + ), + } + + first, diagnostics, err := PlanRecovery(inputs) + if err != nil || len(diagnostics) != 0 { + t.Fatalf("PlanRecovery first error=%v diagnostics=%+v", err, diagnostics) + } + second, diagnostics, err := PlanRecovery(reordered) + if err != nil || len(diagnostics) != 0 { + t.Fatalf("PlanRecovery second error=%v diagnostics=%+v", err, diagnostics) + } + if !reflect.DeepEqual(first, second) { + t.Fatalf("plans differ after input row reorder:\nfirst=%+v\nsecond=%+v", first, second) + } + + gotOrder := make([]string, 0, len(first.Records)) + for _, record := range first.Records { + if record.Record.UUID != nil && *record.Record.UUID != "" { + gotOrder = append(gotOrder, "uuid:"+*record.Record.UUID) + continue + } + gotOrder = append(gotOrder, fmt.Sprintf("path:%s:%d", record.Record.SourcePath, record.Record.Ordinal)) + } + wantOrder := []string{ + "path:/path/a:1", + "path:/path/z:9", + "uuid:" + uuidA, + "uuid:" + uuidB, + "uuid:" + uuidC, + } + if !reflect.DeepEqual(gotOrder, wantOrder) { + t.Fatalf("record order = %#v, want %#v", gotOrder, wantOrder) + } +} + +func recoveryInput(label string, appliedVersion int, records ...models.IndexedRecord) RecoveryInput { + return RecoveryInput{ + Shape: SchemaShape{ + AppliedVersion: appliedVersion, + Signature: "sha256:" + label, + }, + Records: records, + RowCount: len(records), + } +} + +func splitAcrossActiveAndStranded(records []models.IndexedRecord) []RecoveryInput { + if len(records) == 0 { + return []RecoveryInput{recoveryInput("active", 13), recoveryInput("stranded", 7)} + } + return []RecoveryInput{ + recoveryInput("active", 13, records[0]), + recoveryInput("stranded", 7, records[1:]...), + } +} + +func recordWithUUID(uuid, sourcePath string, ordinal int64) models.IndexedRecord { + record := baseRecord(sourcePath, ordinal) + record.UUID = strPtr(uuid) + return record +} + +func recordWithoutUUID(sourcePath string, ordinal int64, uuid *string) models.IndexedRecord { + record := baseRecord(sourcePath, ordinal) + record.UUID = uuid + return record +} + +func baseRecord(sourcePath string, ordinal int64) models.IndexedRecord { + project := "project" + timestamp := "2026-08-18T23:00:00Z" + return models.IndexedRecord{ + Source: "claude", + SourcePath: sourcePath, + Ordinal: ordinal, + Role: "user", + Text: "hello recovery", + Project: &project, + Timestamp: ×tamp, + ContentType: "text/plain", + } +} + +func withText(record models.IndexedRecord, text string) models.IndexedRecord { + record.Text = text + return record +} + +func withProject(record models.IndexedRecord, project *string) models.IndexedRecord { + record.Project = project + return record +} + +func withSourceAndRole(record models.IndexedRecord, source, role string) models.IndexedRecord { + record.Source = source + record.Role = role + return record +} + +func strPtr(value string) *string { + return &value +} + +func assertInputShapes(t *testing.T, plan RecoveryPlan, inputs []RecoveryInput) { + t.Helper() + if len(plan.InputShapes) != len(inputs) { + t.Fatalf("input shape count = %d, want %d", len(plan.InputShapes), len(inputs)) + } + for i, input := range inputs { + if plan.InputShapes[i] != input.Shape { + t.Fatalf("input shape %d = %+v, want %+v", i, plan.InputShapes[i], input.Shape) + } + } +} + +func assertDiagnosticCounts(t *testing.T, diagnostics []Diagnostic, want map[Code]int) { + t.Helper() + got := map[Code]int{} + for _, diagnostic := range diagnostics { + got[diagnostic.Code]++ + } + if len(got) == 0 && len(want) == 0 { + return + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("diagnostic counts = %#v from diagnostics %+v, want %#v", got, diagnostics, want) + } +} + +func assertDiagnosticSummariesContain(t *testing.T, diagnostics []Diagnostic, parts []string) { + t.Helper() + joined := make([]string, 0, len(diagnostics)) + for _, diagnostic := range diagnostics { + if diagnostic.Summary == "" { + t.Fatalf("empty diagnostic summary in %+v", diagnostics) + } + joined = append(joined, diagnostic.Summary) + } + all := strings.Join(joined, "\n") + for _, part := range parts { + if !strings.Contains(all, part) { + t.Fatalf("diagnostic summaries %q do not contain %q", all, part) + } + } +} diff --git a/internal/compat/schema.go b/internal/compat/schema.go new file mode 100644 index 0000000..dd02b30 --- /dev/null +++ b/internal/compat/schema.go @@ -0,0 +1,424 @@ +package compat + +import ( + "context" + "crypto/sha256" + "database/sql" + "errors" + "fmt" + "sort" + "strings" +) + +var ( + defaultCatalog Catalog + defaultCatalogErr error +) + +func init() { + defaultCatalog, defaultCatalogErr = LoadCatalog() +} + +func InspectIndex(ctx context.Context, q Queryer) (MigrationPlan, *Diagnostic, error) { + shape, err := inspectShape(ctx, q) + if err != nil { + return MigrationPlan{}, nil, fmt.Errorf("inspect schema: %w", err) + } + plan := MigrationPlan{From: shape.SchemaShape} + if defaultCatalogErr != nil { + return plan, nil, fmt.Errorf("load schema catalog: %w", defaultCatalogErr) + } + lineage, ok := defaultCatalog.BySignature(shape.Signature) + if !ok { + return plan, &Diagnostic{ + Code: CodeUnsupportedLineage, + Summary: fmt.Sprintf("unsupported index schema %s", shape.Signature), + }, nil + } + plan.Steps = lineage.RemainingSteps() + return plan, nil, nil +} + +func VerifyCurrentShape(ctx context.Context, q Queryer) error { + plan, diag, err := InspectIndex(ctx, q) + if err != nil { + return err + } + if diag != nil { + return fmt.Errorf("%s: %s", diag.Code, diag.Summary) + } + if len(plan.Steps) != 0 { + return fmt.Errorf("index schema %s has %d pending migration step(s)", plan.From.Signature, len(plan.Steps)) + } + return nil +} + +type inspectedShape struct { + SchemaShape + columnsByTable map[string]map[string]bool +} + +func inspectShape(ctx context.Context, q Queryer) (inspectedShape, error) { + objects, err := loadSQLiteObjects(ctx, q) + if err != nil { + return inspectedShape{}, err + } + + columnsByTable := map[string]map[string]bool{} + var records []string + appliedVersion := 0 + + if hasObject(objects, "table", "schema_migrations") { + migrationRows, maxVersion, err := loadSchemaMigrationRecords(ctx, q) + if err != nil { + return inspectedShape{}, err + } + records = append(records, migrationRows...) + appliedVersion = maxVersion + } + + virtualTables := map[string]bool{} + for _, object := range objects { + if object.typ == "table" && isVirtualTableSQL(object.sql) { + virtualTables[object.name] = true + } + } + + for _, object := range objects { + if isVolatileSQLiteObject(object.name) || isFTSShadowObject(object.name, virtualTables) { + continue + } + if object.typ == "table" && !virtualTables[object.name] { + tableRecords, columns, err := loadRegularTableRecords(ctx, q, object) + if err != nil { + return inspectedShape{}, err + } + records = append(records, tableRecords...) + for _, column := range columns { + if columnsByTable[object.name] == nil { + columnsByTable[object.name] = map[string]bool{} + } + columnsByTable[object.name][column.name] = true + } + continue + } + records = append(records, schemaRecord(object.typ, object.table, object.name, "", normalizeSQL(object.sql))) + if object.typ != "table" { + continue + } + columns, err := loadTableColumns(ctx, q, object.name) + if err != nil { + return inspectedShape{}, err + } + for _, column := range columns { + records = append(records, schemaRecord("column", object.name, column.name, column.signature(), "")) + if columnsByTable[object.name] == nil { + columnsByTable[object.name] = map[string]bool{} + } + columnsByTable[object.name][column.name] = true + } + indexRecords, err := loadIndexRecords(ctx, q, object.name) + if err != nil { + return inspectedShape{}, err + } + records = append(records, indexRecords...) + } + + sort.Strings(records) + signatureBytes := sha256.Sum256([]byte(strings.Join(records, "\n"))) + return inspectedShape{ + SchemaShape: SchemaShape{ + AppliedVersion: appliedVersion, + Signature: fmt.Sprintf("sha256:%x", signatureBytes), + }, + columnsByTable: columnsByTable, + }, nil +} + +type sqliteObject struct { + typ string + name string + table string + sql string +} + +type schemaRowsCloser interface { + Close() error +} + +func joinRowsCloseError(rows schemaRowsCloser, err *error) { + if rows == nil || err == nil { + return + } + if closeErr := rows.Close(); closeErr != nil { + *err = errors.Join(*err, closeErr) + } +} + +func loadSQLiteObjects(ctx context.Context, q Queryer) (objects []sqliteObject, err error) { + rows, err := q.QueryContext(ctx, ` + SELECT type, name, tbl_name, COALESCE(sql, '') + FROM sqlite_master + ORDER BY type, name + `) + if err != nil { + return nil, fmt.Errorf("query sqlite_master: %w", err) + } + defer joinRowsCloseError(rows, &err) + + for rows.Next() { + var object sqliteObject + if scanErr := rows.Scan(&object.typ, &object.name, &object.table, &object.sql); scanErr != nil { + err = fmt.Errorf("scan sqlite_master: %w", scanErr) + return nil, err + } + objects = append(objects, object) + } + if rowsErr := rows.Err(); rowsErr != nil { + err = fmt.Errorf("read sqlite_master: %w", rowsErr) + return nil, err + } + return objects, nil +} + +func loadRegularTableRecords(ctx context.Context, q Queryer, object sqliteObject) ([]string, []tableColumn, error) { + columns, err := loadTableColumns(ctx, q, object.name) + if err != nil { + return nil, nil, err + } + + // Regular tables are intentionally signed with their full SQLite DDL evidence. + // Earlier semantic canonicalization accepted unsupported altered tables by + // collision because PRAGMA-derived fields omit DDL semantics such as + // AUTOINCREMENT, COLLATE, ON CONFLICT, and DEFERRABLE. The only non-canonical + // altered shape we support is an explicit checked-in fixture/signature. + records := []string{schemaRecord("table", object.table, object.name, "", normalizeSQL(object.sql))} + for _, column := range columns { + records = append(records, schemaRecord("column", object.name, column.name, column.signature(), "")) + } + indexRecords, err := loadIndexRecords(ctx, q, object.name) + if err != nil { + return nil, nil, err + } + records = append(records, indexRecords...) + return records, columns, nil +} + +func loadSchemaMigrationRecords(ctx context.Context, q Queryer) (records []string, maxVersion int, err error) { + rows, err := q.QueryContext(ctx, ` + SELECT version, name, checksum + FROM schema_migrations + ORDER BY version + `) + if err != nil { + return nil, 0, fmt.Errorf("query schema_migrations: %w", err) + } + defer joinRowsCloseError(rows, &err) + + for rows.Next() { + var version int + var name, checksum string + if scanErr := rows.Scan(&version, &name, &checksum); scanErr != nil { + err = fmt.Errorf("scan schema_migrations: %w", scanErr) + return nil, 0, err + } + if version > maxVersion { + maxVersion = version + } + records = append(records, schemaRecord("migration", "schema_migrations", fmt.Sprintf("%013d", version), name+"|"+checksum, "")) + } + if rowsErr := rows.Err(); rowsErr != nil { + err = fmt.Errorf("read schema_migrations: %w", rowsErr) + return nil, 0, err + } + return records, maxVersion, nil +} + +type tableColumn struct { + cid int + name string + typ string + notNull int + defaultTo sql.NullString + pk int + hidden int +} + +func (c tableColumn) signature() string { + defaultValue := "" + if c.defaultTo.Valid { + defaultValue = normalizeSQL(c.defaultTo.String) + } + return fmt.Sprintf("%013d:%s:%d:%s:%d:%d", c.cid, c.typ, c.notNull, defaultValue, c.pk, c.hidden) +} + +func loadTableColumns(ctx context.Context, q Queryer, table string) (columns []tableColumn, err error) { + rows, err := q.QueryContext(ctx, "PRAGMA table_xinfo("+quoteIdent(table)+")") + if err != nil { + return nil, fmt.Errorf("query table_info %s: %w", table, err) + } + defer joinRowsCloseError(rows, &err) + + for rows.Next() { + var column tableColumn + if scanErr := rows.Scan(&column.cid, &column.name, &column.typ, &column.notNull, &column.defaultTo, &column.pk, &column.hidden); scanErr != nil { + err = fmt.Errorf("scan table_info %s: %w", table, scanErr) + return nil, err + } + columns = append(columns, column) + } + if rowsErr := rows.Err(); rowsErr != nil { + err = fmt.Errorf("read table_info %s: %w", table, rowsErr) + return nil, err + } + return columns, nil +} + +type sqliteIndex struct { + seq int + name string + unique int + origin string + partial int +} + +func loadIndexRecords(ctx context.Context, q Queryer, table string) ([]string, error) { + indexes, err := loadIndexes(ctx, q, table) + if err != nil { + return nil, err + } + var records []string + for _, index := range indexes { + columns, err := loadIndexColumns(ctx, q, index.name) + if err != nil { + return nil, err + } + metadata := fmt.Sprintf("unique=%d origin=%s partial=%d columns=%s", index.unique, index.origin, index.partial, strings.Join(columns, ",")) + records = append(records, schemaRecord("index", table, index.name, metadata, "")) + } + return records, nil +} + +func loadIndexes(ctx context.Context, q Queryer, table string) (indexes []sqliteIndex, err error) { + rows, err := q.QueryContext(ctx, "PRAGMA index_list("+quoteIdent(table)+")") + if err != nil { + return nil, fmt.Errorf("query index_list %s: %w", table, err) + } + defer joinRowsCloseError(rows, &err) + + for rows.Next() { + var index sqliteIndex + if scanErr := rows.Scan(&index.seq, &index.name, &index.unique, &index.origin, &index.partial); scanErr != nil { + err = fmt.Errorf("scan index_list %s: %w", table, scanErr) + return nil, err + } + indexes = append(indexes, index) + } + if rowsErr := rows.Err(); rowsErr != nil { + err = fmt.Errorf("read index_list %s: %w", table, rowsErr) + return nil, err + } + sort.Slice(indexes, func(i, j int) bool { return indexes[i].name < indexes[j].name }) + return indexes, nil +} + +func loadIndexColumns(ctx context.Context, q Queryer, index string) (columns []string, err error) { + rows, err := q.QueryContext(ctx, "PRAGMA index_info("+quoteIdent(index)+")") + if err != nil { + return nil, fmt.Errorf("query index_info %s: %w", index, err) + } + defer joinRowsCloseError(rows, &err) + + for rows.Next() { + var seqno, cid int + var name sql.NullString + if scanErr := rows.Scan(&seqno, &cid, &name); scanErr != nil { + err = fmt.Errorf("scan index_info %s: %w", index, scanErr) + return nil, err + } + columnName := "" + if name.Valid { + columnName = name.String + } + columns = append(columns, fmt.Sprintf("%013d:%013d:%s", seqno, cid, columnName)) + } + if rowsErr := rows.Err(); rowsErr != nil { + err = fmt.Errorf("read index_info %s: %w", index, rowsErr) + return nil, err + } + sort.Strings(columns) + return columns, nil +} + +func remainingStepsFor(appliedVersion int, hasSourceMetadata bool) []MigrationStep { + var steps []MigrationStep + for _, step := range allMigrationSteps { + if step.Version <= appliedVersion { + continue + } + if step.Version == 6 && !hasSourceMetadata { + continue + } + steps = append(steps, step) + } + return steps +} + +var allMigrationSteps = []MigrationStep{ + {Version: 1, Name: "V1 core schema"}, + {Version: 2, Name: "V2 embedding tables"}, + {Version: 3, Name: "V3 embedding blob column"}, + {Version: 4, Name: "V4 tool_fts trigram index"}, + {Version: 5, Name: "V5 drop phantom session_events"}, + {Version: 6, Name: "V6 drop source_metadata when present"}, + {Version: 7, Name: "V7 reasoning triggers"}, + {Version: 8, Name: "V8 perennity: extraction_version, was_interrupted, tool_events"}, + {Version: 9, Name: "V9 tool_events uuid uniqueness index"}, + {Version: 10, Name: "V10 template mining: message_templates, template_matches"}, + {Version: 11, Name: "V11 correction detection: correction_signals"}, + {Version: 12, Name: "V12 agent classification: annotations"}, + {Version: 13, Name: "V13 backfill discovery indexes"}, +} + +func hasObject(objects []sqliteObject, typ, name string) bool { + for _, object := range objects { + if object.typ == typ && object.name == name { + return true + } + } + return false +} + +func isVolatileSQLiteObject(name string) bool { + return strings.HasPrefix(name, "sqlite_") +} + +func isVirtualTableSQL(sqlText string) bool { + return strings.Contains(strings.ToLower(sqlText), " using fts5") +} + +func isFTSShadowObject(name string, virtualTables map[string]bool) bool { + for virtualTable := range virtualTables { + if name == virtualTable { + continue + } + for _, suffix := range []string{"_data", "_idx", "_content", "_docsize", "_config"} { + if name == virtualTable+suffix { + return true + } + } + } + return false +} + +func normalizeSQL(sqlText string) string { + return strings.TrimSpace(sqlText) +} + +func schemaRecord(kind, table, name, columns, sqlText string) string { + return strings.Join([]string{kind, table, name, columns, sqlText}, "|") +} + +func quoteIdent(identifier string) string { + return `"` + strings.ReplaceAll(identifier, `"`, `""`) + `"` +} diff --git a/internal/compat/schema_test.go b/internal/compat/schema_test.go new file mode 100644 index 0000000..3477a7b --- /dev/null +++ b/internal/compat/schema_test.go @@ -0,0 +1,351 @@ +package compat + +import ( + "context" + "database/sql" + "errors" + "io/fs" + "strings" + "testing" + + _ "modernc.org/sqlite" +) + +func TestInspectIndexUsesObservedShapeNotVersionAlone(t *testing.T) { + tests := []struct { + fixture string + wantFirstStep string + }{ + {"v5-with-source-metadata.sql", "V6 drop source_metadata when present"}, + {"v5-without-source-metadata.sql", "V7 reasoning triggers"}, + } + + for _, tt := range tests { + t.Run(tt.fixture, func(t *testing.T) { + db := openFixtureCopy(t, tt.fixture) + defer db.Close() + + plan, diag, err := InspectIndex(context.Background(), db) + if err != nil || diag != nil { + t.Fatalf("plan error=%v diagnostic=%+v", err, diag) + } + if len(plan.Steps) == 0 || plan.Steps[0].Name != tt.wantFirstStep { + t.Fatalf("steps=%+v", plan.Steps) + } + }) + } +} + +func TestInspectIndexRecognizesPartialV6LineageAndPlansV7(t *testing.T) { + db := openFixtureCopy(t, "v6.sql") + defer db.Close() + + plan, diag, err := InspectIndex(context.Background(), db) + if err != nil || diag != nil { + t.Fatalf("plan error=%v diagnostic=%+v", err, diag) + } + if plan.From.AppliedVersion != 6 { + t.Fatalf("applied version = %d, want 6", plan.From.AppliedVersion) + } + if len(plan.Steps) == 0 || plan.Steps[0].Name != "V7 reasoning triggers" { + t.Fatalf("steps=%+v, want V7+", plan.Steps) + } +} + +func TestInspectIndexCurrentShapeIsIdempotent(t *testing.T) { + db := openFixtureCopy(t, "v13.sql") + defer db.Close() + + plan, diag, err := InspectIndex(context.Background(), db) + if err != nil || diag != nil { + t.Fatalf("plan error=%v diagnostic=%+v", err, diag) + } + if plan.From.AppliedVersion != 13 { + t.Fatalf("applied version = %d, want 13", plan.From.AppliedVersion) + } + if len(plan.Steps) != 0 { + t.Fatalf("current shape has pending steps: %+v", plan.Steps) + } + if err := VerifyCurrentShape(context.Background(), db); err != nil { + t.Fatalf("verify current shape: %v", err) + } +} + +func TestInspectIndexUnsupportedShapeReturnsInternalDiagnostic(t *testing.T) { + db := openFixtureCopy(t, "v13.sql") + defer db.Close() + if _, err := db.Exec("CREATE TABLE unexpected_shape_marker (id INTEGER PRIMARY KEY)"); err != nil { + t.Fatal(err) + } + + plan, diag, err := InspectIndex(context.Background(), db) + if err != nil { + t.Fatalf("inspect error = %v", err) + } + if diag == nil { + t.Fatalf("diagnostic is nil; plan=%+v", plan) + } + if diag.Code != CodeUnsupportedLineage { + t.Fatalf("diagnostic code = %s, want %s", diag.Code, CodeUnsupportedLineage) + } + if !strings.Contains(diag.Summary, plan.From.Signature) { + t.Fatalf("summary %q does not include signature %q", diag.Summary, plan.From.Signature) + } + if len(diag.Continuation) != 0 { + t.Fatalf("continuation = %+v, want empty", diag.Continuation) + } +} + +func TestVerifyCurrentShapeRejectsPendingMigrations(t *testing.T) { + db := openFixtureCopy(t, "v5-without-source-metadata.sql") + defer db.Close() + + if err := VerifyCurrentShape(context.Background(), db); err == nil { + t.Fatal("verify current shape succeeded for a shape with pending migrations") + } +} + +func TestJoinRowsCloseErrorJoinsPrimaryAndCloseErrors(t *testing.T) { + primaryErr := errors.New("primary schema rows failure") + closeErr := errors.New("close schema rows failure") + err := primaryErr + + joinRowsCloseError(fakeSchemaRowsCloser{err: closeErr}, &err) + + if err == nil || !errors.Is(err, primaryErr) || !errors.Is(err, closeErr) { + t.Fatalf("joined error = %v, want primary and close", err) + } +} + +func TestJoinRowsCloseErrorLeavesSuccessfulScansUnchanged(t *testing.T) { + var err error + joinRowsCloseError(fakeSchemaRowsCloser{}, &err) + if err != nil { + t.Fatalf("successful rows close error = %v, want nil", err) + } + + primaryErr := errors.New("primary schema rows failure") + err = primaryErr + joinRowsCloseError(fakeSchemaRowsCloser{}, &err) + if err != primaryErr { + t.Fatalf("primary error changed to %v", err) + } +} + +type fakeSchemaRowsCloser struct { + err error +} + +func (r fakeSchemaRowsCloser) Close() error { return r.err } + +func TestInspectIndexMalformedMigrationMetadataReturnsError(t *testing.T) { + db := openSchema(t, ` + CREATE TABLE schema_migrations (version INTEGER PRIMARY KEY); + `) + defer db.Close() + + _, diag, err := InspectIndex(context.Background(), db) + if err == nil { + t.Fatalf("inspect succeeded with diagnostic %+v", diag) + } + if diag != nil { + t.Fatalf("diagnostic = %+v, want nil Go error path", diag) + } + if !strings.Contains(err.Error(), "inspect schema:") || !strings.Contains(err.Error(), "schema_migrations") { + t.Fatalf("error = %v, want wrapped schema_migrations error", err) + } +} + +func TestRegularTableSignatureIsConservativeForUnsupportedDDL(t *testing.T) { + for _, tt := range []struct { + name string + left string + right string + }{ + { + name: "generated column", + left: `CREATE TABLE items (id INTEGER PRIMARY KEY, body TEXT NOT NULL);`, + right: `CREATE TABLE items (id INTEGER PRIMARY KEY, body TEXT NOT NULL, body_len INTEGER GENERATED ALWAYS AS (length(body)) VIRTUAL);`, + }, + { + name: "autoincrement", + left: `CREATE TABLE items (id INTEGER PRIMARY KEY, body TEXT NOT NULL);`, + right: `CREATE TABLE items (id INTEGER PRIMARY KEY AUTOINCREMENT, body TEXT NOT NULL);`, + }, + { + name: "collation", + left: `CREATE TABLE items (id INTEGER PRIMARY KEY, body TEXT NOT NULL);`, + right: `CREATE TABLE items (id INTEGER PRIMARY KEY, body TEXT COLLATE NOCASE NOT NULL);`, + }, + { + name: "on conflict", + left: `CREATE TABLE items (id INTEGER PRIMARY KEY, body TEXT UNIQUE NOT NULL);`, + right: `CREATE TABLE items (id INTEGER PRIMARY KEY, body TEXT UNIQUE ON CONFLICT REPLACE NOT NULL);`, + }, + { + name: "deferrable foreign key", + left: ` + CREATE TABLE parents (id INTEGER PRIMARY KEY); + CREATE TABLE items (id INTEGER PRIMARY KEY, parent_id INTEGER NOT NULL REFERENCES parents(id)); + `, + right: ` + CREATE TABLE parents (id INTEGER PRIMARY KEY); + CREATE TABLE items (id INTEGER PRIMARY KEY, parent_id INTEGER NOT NULL REFERENCES parents(id) DEFERRABLE INITIALLY DEFERRED); + `, + }, + } { + t.Run(tt.name, func(t *testing.T) { + left := openSchema(t, tt.left) + defer left.Close() + right := openSchema(t, tt.right) + defer right.Close() + + leftShape, err := inspectShape(context.Background(), left) + if err != nil { + t.Fatal(err) + } + rightShape, err := inspectShape(context.Background(), right) + if err != nil { + t.Fatal(err) + } + if leftShape.Signature == rightShape.Signature { + t.Fatalf("%s DDL difference collided at signature %s", tt.name, leftShape.Signature) + } + }) + } +} + +func TestConservativeSignaturePreservesWhitespaceInsideQuotedSQL(t *testing.T) { + for _, tt := range []struct { + name string + left string + right string + }{ + { + name: "default quoted literal", + left: `CREATE TABLE items (id INTEGER PRIMARY KEY, body TEXT DEFAULT 'alpha beta');`, + right: `CREATE TABLE items (id INTEGER PRIMARY KEY, body TEXT DEFAULT 'alpha beta');`, + }, + { + name: "check quoted literal", + left: `CREATE TABLE items (id INTEGER PRIMARY KEY, body TEXT CHECK (body <> 'alpha beta'));`, + right: `CREATE TABLE items (id INTEGER PRIMARY KEY, body TEXT CHECK (body <> 'alpha beta'));`, + }, + } { + t.Run(tt.name, func(t *testing.T) { + left := openSchema(t, tt.left) + defer left.Close() + right := openSchema(t, tt.right) + defer right.Close() + + leftShape, err := inspectShape(context.Background(), left) + if err != nil { + t.Fatal(err) + } + rightShape, err := inspectShape(context.Background(), right) + if err != nil { + t.Fatal(err) + } + if leftShape.Signature == rightShape.Signature { + t.Fatalf("quoted whitespace difference collided at signature %s", leftShape.Signature) + } + }) + } +} + +func TestConservativeSignatureStableForExactSameSQL(t *testing.T) { + schemaSQL := `CREATE TABLE items (id INTEGER PRIMARY KEY, body TEXT DEFAULT 'alpha beta' CHECK (body <> 'gamma delta'));` + left := openSchema(t, schemaSQL) + defer left.Close() + right := openSchema(t, schemaSQL) + defer right.Close() + + leftShape, err := inspectShape(context.Background(), left) + if err != nil { + t.Fatal(err) + } + rightShape, err := inspectShape(context.Background(), right) + if err != nil { + t.Fatal(err) + } + if leftShape.Signature != rightShape.Signature { + t.Fatalf("exact same SQL signature differs: %s != %s", leftShape.Signature, rightShape.Signature) + } +} + +func TestInspectIndexRecognizesCanonicalAndExplicitLegacyV13Shapes(t *testing.T) { + for _, fixture := range []string{"v13.sql", "v13-legacy-alter-built.sql", "v13-legacy-existing-schema-migrations.sql"} { + t.Run(fixture, func(t *testing.T) { + db := openFixtureCopy(t, fixture) + defer db.Close() + + plan, diag, err := InspectIndex(context.Background(), db) + if err != nil || diag != nil { + t.Fatalf("inspect error=%v diagnostic=%+v", err, diag) + } + if plan.From.AppliedVersion != 13 { + t.Fatalf("applied version = %d, want 13", plan.From.AppliedVersion) + } + if len(plan.Steps) != 0 { + t.Fatalf("V13-compatible shape has pending steps: %+v", plan.Steps) + } + }) + } +} + +func TestInspectShapeSignatureIsStableAcrossMetadataOrder(t *testing.T) { + left := openSchema(t, ` + CREATE TABLE schema_migrations (version INTEGER PRIMARY KEY, name TEXT NOT NULL, applied_on TEXT NOT NULL, checksum TEXT NOT NULL); + INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (1, 'one', 'first clock', 'checksum'); + CREATE TABLE alpha (id INTEGER PRIMARY KEY, body TEXT NOT NULL); + CREATE INDEX idx_alpha_body ON alpha(body); + CREATE TABLE beta (id INTEGER PRIMARY KEY, alpha_id INTEGER NOT NULL); + CREATE INDEX idx_beta_alpha ON beta(alpha_id); + `) + defer left.Close() + right := openSchema(t, ` + CREATE TABLE beta (id INTEGER PRIMARY KEY, alpha_id INTEGER NOT NULL); + CREATE INDEX idx_beta_alpha ON beta(alpha_id); + CREATE TABLE schema_migrations (version INTEGER PRIMARY KEY, name TEXT NOT NULL, applied_on TEXT NOT NULL, checksum TEXT NOT NULL); + INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (1, 'one', 'second clock', 'checksum'); + CREATE TABLE alpha (id INTEGER PRIMARY KEY, body TEXT NOT NULL); + CREATE INDEX idx_alpha_body ON alpha(body); + `) + defer right.Close() + + leftShape, err := inspectShape(context.Background(), left) + if err != nil { + t.Fatal(err) + } + rightShape, err := inspectShape(context.Background(), right) + if err != nil { + t.Fatal(err) + } + if leftShape.Signature != rightShape.Signature { + t.Fatalf("signatures differ after metadata reordering: %s != %s", leftShape.Signature, rightShape.Signature) + } +} + +func openFixtureCopy(t *testing.T, fixture string) *sql.DB { + t.Helper() + + data, err := fs.ReadFile(releaseSchemaFS, "testdata/release-schemas/"+fixture) + if err != nil { + t.Fatal(err) + } + return openSchema(t, string(data)) +} + +func openSchema(t *testing.T, schemaSQL string) *sql.DB { + t.Helper() + + db, err := sql.Open("sqlite", ":memory:") + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(schemaSQL); err != nil { + db.Close() + t.Fatal(err) + } + return db +} diff --git a/internal/compat/testdata/release-schemas/manifest.json b/internal/compat/testdata/release-schemas/manifest.json new file mode 100644 index 0000000..a14fdef --- /dev/null +++ b/internal/compat/testdata/release-schemas/manifest.json @@ -0,0 +1,608 @@ +{ + "FirstGoRelease": "v0.3.7", + "LatestGoRelease": "v3.2.5", + "Releases": [ + { + "Tag": "v0.3.7", + "Fixture": "v1.sql", + "ProvenanceSHA256": "7b09c4c93e90b0cbd45cc0b851dcacefc0590145582886636c2e420145ccb409", + "Signature": "sha256:e2f7cd4bd71c964717c00fd67c2b3f396306307f578382c4a7956f83f8555a57", + "AppliedVersion": 1, + "HasSourceMetadata": true + }, + { + "Tag": "v0.3.9", + "Fixture": "v1.sql", + "ProvenanceSHA256": "7b09c4c93e90b0cbd45cc0b851dcacefc0590145582886636c2e420145ccb409", + "Signature": "sha256:e2f7cd4bd71c964717c00fd67c2b3f396306307f578382c4a7956f83f8555a57", + "AppliedVersion": 1, + "HasSourceMetadata": true + }, + { + "Tag": "v0.3.10", + "Fixture": "v1.sql", + "ProvenanceSHA256": "7b09c4c93e90b0cbd45cc0b851dcacefc0590145582886636c2e420145ccb409", + "Signature": "sha256:e2f7cd4bd71c964717c00fd67c2b3f396306307f578382c4a7956f83f8555a57", + "AppliedVersion": 1, + "HasSourceMetadata": true + }, + { + "Tag": "v0.3.11", + "Fixture": "v3.sql", + "ProvenanceSHA256": "1882e91edfc1eaf865b27b1796b8d20c92d05a2a55920f37f428df8b4e348e5d", + "Signature": "sha256:ed864d83d4c873bd34e0e3708f32a5bf47d65cac8ad55ff04305a81d91e9e22e", + "AppliedVersion": 3, + "HasSourceMetadata": true + }, + { + "Tag": "v0.3.12", + "Fixture": "v3.sql", + "ProvenanceSHA256": "1882e91edfc1eaf865b27b1796b8d20c92d05a2a55920f37f428df8b4e348e5d", + "Signature": "sha256:ed864d83d4c873bd34e0e3708f32a5bf47d65cac8ad55ff04305a81d91e9e22e", + "AppliedVersion": 3, + "HasSourceMetadata": true + }, + { + "Tag": "v0.3.13", + "Fixture": "v3.sql", + "ProvenanceSHA256": "1882e91edfc1eaf865b27b1796b8d20c92d05a2a55920f37f428df8b4e348e5d", + "Signature": "sha256:ed864d83d4c873bd34e0e3708f32a5bf47d65cac8ad55ff04305a81d91e9e22e", + "AppliedVersion": 3, + "HasSourceMetadata": true + }, + { + "Tag": "v0.3.14", + "Fixture": "v3.sql", + "ProvenanceSHA256": "1882e91edfc1eaf865b27b1796b8d20c92d05a2a55920f37f428df8b4e348e5d", + "Signature": "sha256:ed864d83d4c873bd34e0e3708f32a5bf47d65cac8ad55ff04305a81d91e9e22e", + "AppliedVersion": 3, + "HasSourceMetadata": true + }, + { + "Tag": "v0.3.15", + "Fixture": "v3.sql", + "ProvenanceSHA256": "1882e91edfc1eaf865b27b1796b8d20c92d05a2a55920f37f428df8b4e348e5d", + "Signature": "sha256:ed864d83d4c873bd34e0e3708f32a5bf47d65cac8ad55ff04305a81d91e9e22e", + "AppliedVersion": 3, + "HasSourceMetadata": true + }, + { + "Tag": "v0.3.17", + "Fixture": "v3.sql", + "ProvenanceSHA256": "1882e91edfc1eaf865b27b1796b8d20c92d05a2a55920f37f428df8b4e348e5d", + "Signature": "sha256:ed864d83d4c873bd34e0e3708f32a5bf47d65cac8ad55ff04305a81d91e9e22e", + "AppliedVersion": 3, + "HasSourceMetadata": true + }, + { + "Tag": "v0.3.18", + "Fixture": "v3.sql", + "ProvenanceSHA256": "1882e91edfc1eaf865b27b1796b8d20c92d05a2a55920f37f428df8b4e348e5d", + "Signature": "sha256:ed864d83d4c873bd34e0e3708f32a5bf47d65cac8ad55ff04305a81d91e9e22e", + "AppliedVersion": 3, + "HasSourceMetadata": true + }, + { + "Tag": "v0.3.19", + "Fixture": "v3.sql", + "ProvenanceSHA256": "1882e91edfc1eaf865b27b1796b8d20c92d05a2a55920f37f428df8b4e348e5d", + "Signature": "sha256:ed864d83d4c873bd34e0e3708f32a5bf47d65cac8ad55ff04305a81d91e9e22e", + "AppliedVersion": 3, + "HasSourceMetadata": true + }, + { + "Tag": "v0.4.0", + "Fixture": "v3.sql", + "ProvenanceSHA256": "1882e91edfc1eaf865b27b1796b8d20c92d05a2a55920f37f428df8b4e348e5d", + "Signature": "sha256:ed864d83d4c873bd34e0e3708f32a5bf47d65cac8ad55ff04305a81d91e9e22e", + "AppliedVersion": 3, + "HasSourceMetadata": true + }, + { + "Tag": "v0.4.1", + "Fixture": "v3.sql", + "ProvenanceSHA256": "1882e91edfc1eaf865b27b1796b8d20c92d05a2a55920f37f428df8b4e348e5d", + "Signature": "sha256:ed864d83d4c873bd34e0e3708f32a5bf47d65cac8ad55ff04305a81d91e9e22e", + "AppliedVersion": 3, + "HasSourceMetadata": true + }, + { + "Tag": "v0.4.3", + "Fixture": "v3.sql", + "ProvenanceSHA256": "1882e91edfc1eaf865b27b1796b8d20c92d05a2a55920f37f428df8b4e348e5d", + "Signature": "sha256:ed864d83d4c873bd34e0e3708f32a5bf47d65cac8ad55ff04305a81d91e9e22e", + "AppliedVersion": 3, + "HasSourceMetadata": true + }, + { + "Tag": "v1.0.0", + "Fixture": "v3.sql", + "ProvenanceSHA256": "1882e91edfc1eaf865b27b1796b8d20c92d05a2a55920f37f428df8b4e348e5d", + "Signature": "sha256:ed864d83d4c873bd34e0e3708f32a5bf47d65cac8ad55ff04305a81d91e9e22e", + "AppliedVersion": 3, + "HasSourceMetadata": true + }, + { + "Tag": "v1.0.1", + "Fixture": "v3.sql", + "ProvenanceSHA256": "1882e91edfc1eaf865b27b1796b8d20c92d05a2a55920f37f428df8b4e348e5d", + "Signature": "sha256:ed864d83d4c873bd34e0e3708f32a5bf47d65cac8ad55ff04305a81d91e9e22e", + "AppliedVersion": 3, + "HasSourceMetadata": true + }, + { + "Tag": "v1.0.2", + "Fixture": "v3.sql", + "ProvenanceSHA256": "1882e91edfc1eaf865b27b1796b8d20c92d05a2a55920f37f428df8b4e348e5d", + "Signature": "sha256:ed864d83d4c873bd34e0e3708f32a5bf47d65cac8ad55ff04305a81d91e9e22e", + "AppliedVersion": 3, + "HasSourceMetadata": true + }, + { + "Tag": "v1.0.3", + "Fixture": "v3.sql", + "ProvenanceSHA256": "1882e91edfc1eaf865b27b1796b8d20c92d05a2a55920f37f428df8b4e348e5d", + "Signature": "sha256:ed864d83d4c873bd34e0e3708f32a5bf47d65cac8ad55ff04305a81d91e9e22e", + "AppliedVersion": 3, + "HasSourceMetadata": true + }, + { + "Tag": "v1.0.4", + "Fixture": "v3.sql", + "ProvenanceSHA256": "1882e91edfc1eaf865b27b1796b8d20c92d05a2a55920f37f428df8b4e348e5d", + "Signature": "sha256:ed864d83d4c873bd34e0e3708f32a5bf47d65cac8ad55ff04305a81d91e9e22e", + "AppliedVersion": 3, + "HasSourceMetadata": true + }, + { + "Tag": "v1.1.0", + "Fixture": "v3.sql", + "ProvenanceSHA256": "1882e91edfc1eaf865b27b1796b8d20c92d05a2a55920f37f428df8b4e348e5d", + "Signature": "sha256:ed864d83d4c873bd34e0e3708f32a5bf47d65cac8ad55ff04305a81d91e9e22e", + "AppliedVersion": 3, + "HasSourceMetadata": true + }, + { + "Tag": "v1.1.1", + "Fixture": "v3.sql", + "ProvenanceSHA256": "1882e91edfc1eaf865b27b1796b8d20c92d05a2a55920f37f428df8b4e348e5d", + "Signature": "sha256:ed864d83d4c873bd34e0e3708f32a5bf47d65cac8ad55ff04305a81d91e9e22e", + "AppliedVersion": 3, + "HasSourceMetadata": true + }, + { + "Tag": "v1.2.0", + "Fixture": "v3.sql", + "ProvenanceSHA256": "1882e91edfc1eaf865b27b1796b8d20c92d05a2a55920f37f428df8b4e348e5d", + "Signature": "sha256:ed864d83d4c873bd34e0e3708f32a5bf47d65cac8ad55ff04305a81d91e9e22e", + "AppliedVersion": 3, + "HasSourceMetadata": true + }, + { + "Tag": "v1.2.1", + "Fixture": "v3.sql", + "ProvenanceSHA256": "1882e91edfc1eaf865b27b1796b8d20c92d05a2a55920f37f428df8b4e348e5d", + "Signature": "sha256:ed864d83d4c873bd34e0e3708f32a5bf47d65cac8ad55ff04305a81d91e9e22e", + "AppliedVersion": 3, + "HasSourceMetadata": true + }, + { + "Tag": "v1.3.0", + "Fixture": "v3.sql", + "ProvenanceSHA256": "1882e91edfc1eaf865b27b1796b8d20c92d05a2a55920f37f428df8b4e348e5d", + "Signature": "sha256:ed864d83d4c873bd34e0e3708f32a5bf47d65cac8ad55ff04305a81d91e9e22e", + "AppliedVersion": 3, + "HasSourceMetadata": true + }, + { + "Tag": "v1.3.1", + "Fixture": "v3.sql", + "ProvenanceSHA256": "1882e91edfc1eaf865b27b1796b8d20c92d05a2a55920f37f428df8b4e348e5d", + "Signature": "sha256:ed864d83d4c873bd34e0e3708f32a5bf47d65cac8ad55ff04305a81d91e9e22e", + "AppliedVersion": 3, + "HasSourceMetadata": true + }, + { + "Tag": "v1.3.2", + "Fixture": "v3.sql", + "ProvenanceSHA256": "1882e91edfc1eaf865b27b1796b8d20c92d05a2a55920f37f428df8b4e348e5d", + "Signature": "sha256:ed864d83d4c873bd34e0e3708f32a5bf47d65cac8ad55ff04305a81d91e9e22e", + "AppliedVersion": 3, + "HasSourceMetadata": true + }, + { + "Tag": "v1.3.3", + "Fixture": "v3.sql", + "ProvenanceSHA256": "1882e91edfc1eaf865b27b1796b8d20c92d05a2a55920f37f428df8b4e348e5d", + "Signature": "sha256:ed864d83d4c873bd34e0e3708f32a5bf47d65cac8ad55ff04305a81d91e9e22e", + "AppliedVersion": 3, + "HasSourceMetadata": true + }, + { + "Tag": "v1.3.5", + "Fixture": "v3.sql", + "ProvenanceSHA256": "1882e91edfc1eaf865b27b1796b8d20c92d05a2a55920f37f428df8b4e348e5d", + "Signature": "sha256:ed864d83d4c873bd34e0e3708f32a5bf47d65cac8ad55ff04305a81d91e9e22e", + "AppliedVersion": 3, + "HasSourceMetadata": true + }, + { + "Tag": "v1.4.0", + "Fixture": "v4.sql", + "ProvenanceSHA256": "e8f28b3a1b37acd4f6bd721615c150d53c9d511c05c53e625d4de22230b3667c", + "Signature": "sha256:b2bba3cec75bb3f6c697e78882e3924550832e3b5706d013f3f54d32a1c25acd", + "AppliedVersion": 4, + "HasSourceMetadata": true + }, + { + "Tag": "v1.4.1", + "Fixture": "v4.sql", + "ProvenanceSHA256": "e8f28b3a1b37acd4f6bd721615c150d53c9d511c05c53e625d4de22230b3667c", + "Signature": "sha256:b2bba3cec75bb3f6c697e78882e3924550832e3b5706d013f3f54d32a1c25acd", + "AppliedVersion": 4, + "HasSourceMetadata": true + }, + { + "Tag": "v1.4.2", + "Fixture": "v4.sql", + "ProvenanceSHA256": "e8f28b3a1b37acd4f6bd721615c150d53c9d511c05c53e625d4de22230b3667c", + "Signature": "sha256:b2bba3cec75bb3f6c697e78882e3924550832e3b5706d013f3f54d32a1c25acd", + "AppliedVersion": 4, + "HasSourceMetadata": true + }, + { + "Tag": "v1.4.3", + "Fixture": "v4.sql", + "ProvenanceSHA256": "e8f28b3a1b37acd4f6bd721615c150d53c9d511c05c53e625d4de22230b3667c", + "Signature": "sha256:b2bba3cec75bb3f6c697e78882e3924550832e3b5706d013f3f54d32a1c25acd", + "AppliedVersion": 4, + "HasSourceMetadata": true + }, + { + "Tag": "v1.4.4", + "Fixture": "v4.sql", + "ProvenanceSHA256": "e8f28b3a1b37acd4f6bd721615c150d53c9d511c05c53e625d4de22230b3667c", + "Signature": "sha256:b2bba3cec75bb3f6c697e78882e3924550832e3b5706d013f3f54d32a1c25acd", + "AppliedVersion": 4, + "HasSourceMetadata": true + }, + { + "Tag": "v2.0.0", + "Fixture": "v5-with-source-metadata.sql", + "ProvenanceSHA256": "1efece456c94cf46a7f5dba8f84e0a76fb444dbd4cae016b595a7fd68427c062", + "Signature": "sha256:ce23ee41f1af6007bd0bde7b020f2cac4c215d23bae456bdfce25b90313c709e", + "AppliedVersion": 5, + "HasSourceMetadata": true + }, + { + "Tag": "v2.0.1", + "Fixture": "v5-with-source-metadata.sql", + "ProvenanceSHA256": "1efece456c94cf46a7f5dba8f84e0a76fb444dbd4cae016b595a7fd68427c062", + "Signature": "sha256:ce23ee41f1af6007bd0bde7b020f2cac4c215d23bae456bdfce25b90313c709e", + "AppliedVersion": 5, + "HasSourceMetadata": true + }, + { + "Tag": "v2.1.0", + "Fixture": "v5-with-source-metadata.sql", + "ProvenanceSHA256": "1efece456c94cf46a7f5dba8f84e0a76fb444dbd4cae016b595a7fd68427c062", + "Signature": "sha256:ce23ee41f1af6007bd0bde7b020f2cac4c215d23bae456bdfce25b90313c709e", + "AppliedVersion": 5, + "HasSourceMetadata": true + }, + { + "Tag": "v2.2.0", + "Fixture": "v7.sql", + "ProvenanceSHA256": "324a876835ad39ebe9749bf1a381018c267f4c21df053589d2046d2eb9cca24e", + "Signature": "sha256:dcaa1205df039fa545aa7ebe672d391acfcbf651869c2603ceef12dab9de00d2", + "AppliedVersion": 7, + "HasSourceMetadata": false + }, + { + "Tag": "v2.2.1", + "Fixture": "v7.sql", + "ProvenanceSHA256": "324a876835ad39ebe9749bf1a381018c267f4c21df053589d2046d2eb9cca24e", + "Signature": "sha256:dcaa1205df039fa545aa7ebe672d391acfcbf651869c2603ceef12dab9de00d2", + "AppliedVersion": 7, + "HasSourceMetadata": false + }, + { + "Tag": "v2.2.2", + "Fixture": "v7.sql", + "ProvenanceSHA256": "324a876835ad39ebe9749bf1a381018c267f4c21df053589d2046d2eb9cca24e", + "Signature": "sha256:dcaa1205df039fa545aa7ebe672d391acfcbf651869c2603ceef12dab9de00d2", + "AppliedVersion": 7, + "HasSourceMetadata": false + }, + { + "Tag": "v2.2.3", + "Fixture": "v7.sql", + "ProvenanceSHA256": "324a876835ad39ebe9749bf1a381018c267f4c21df053589d2046d2eb9cca24e", + "Signature": "sha256:dcaa1205df039fa545aa7ebe672d391acfcbf651869c2603ceef12dab9de00d2", + "AppliedVersion": 7, + "HasSourceMetadata": false + }, + { + "Tag": "v2.3.0", + "Fixture": "v8.sql", + "ProvenanceSHA256": "c66577744d5feacde305382d5159b67390450c18169d3da6fda5e81051b49cf6", + "Signature": "sha256:6d503881ebac89e009644df5bf24cf7c4b1afed334afb37e6ee87d2ca6edae87", + "AppliedVersion": 8, + "HasSourceMetadata": false + }, + { + "Tag": "v2.4.0", + "Fixture": "v9.sql", + "ProvenanceSHA256": "50469b7ae10e1c824b59d094290c2b59bbc31f3924175d2888bb23f76450b2a1", + "Signature": "sha256:e41ae857c862ffa10516681b65bcd8c755484b338a6847dad5c0d770a202670f", + "AppliedVersion": 9, + "HasSourceMetadata": false + }, + { + "Tag": "v2.5.0", + "Fixture": "v9.sql", + "ProvenanceSHA256": "50469b7ae10e1c824b59d094290c2b59bbc31f3924175d2888bb23f76450b2a1", + "Signature": "sha256:e41ae857c862ffa10516681b65bcd8c755484b338a6847dad5c0d770a202670f", + "AppliedVersion": 9, + "HasSourceMetadata": false + }, + { + "Tag": "v2.6.0", + "Fixture": "v10.sql", + "ProvenanceSHA256": "0b6fa1dbe8981705aa68e0a376c874ae7a98588380d2301d108ddcf5f4410abc", + "Signature": "sha256:8e28ce0fe1c5f3cd36f2d64ac7a7996f032e66c0c32468a0a206eb983491388c", + "AppliedVersion": 10, + "HasSourceMetadata": false + }, + { + "Tag": "v2.7.0", + "Fixture": "v11.sql", + "ProvenanceSHA256": "78927a394a0d1cb62651288e7626de4e2983530a3a03b9147440e56b457e26ba", + "Signature": "sha256:975656bb5e894e12bd30aa65bb3366ca2bdeee23f59aca8e133b18a62e7ffad5", + "AppliedVersion": 11, + "HasSourceMetadata": false + }, + { + "Tag": "v2.8.0", + "Fixture": "v12.sql", + "ProvenanceSHA256": "84710f15fcff04567bfe8abe64f337129722080816b7cfbce35740668aacfe41", + "Signature": "sha256:ff136d58048d69f02be857f632750ef3e2daa35fd6ba637f7645986950f83d31", + "AppliedVersion": 12, + "HasSourceMetadata": false + }, + { + "Tag": "v2.9.0", + "Fixture": "v12.sql", + "ProvenanceSHA256": "84710f15fcff04567bfe8abe64f337129722080816b7cfbce35740668aacfe41", + "Signature": "sha256:ff136d58048d69f02be857f632750ef3e2daa35fd6ba637f7645986950f83d31", + "AppliedVersion": 12, + "HasSourceMetadata": false + }, + { + "Tag": "v2.10.0", + "Fixture": "v12.sql", + "ProvenanceSHA256": "84710f15fcff04567bfe8abe64f337129722080816b7cfbce35740668aacfe41", + "Signature": "sha256:ff136d58048d69f02be857f632750ef3e2daa35fd6ba637f7645986950f83d31", + "AppliedVersion": 12, + "HasSourceMetadata": false + }, + { + "Tag": "v2.11.0", + "Fixture": "v12.sql", + "ProvenanceSHA256": "84710f15fcff04567bfe8abe64f337129722080816b7cfbce35740668aacfe41", + "Signature": "sha256:ff136d58048d69f02be857f632750ef3e2daa35fd6ba637f7645986950f83d31", + "AppliedVersion": 12, + "HasSourceMetadata": false + }, + { + "Tag": "v2.12.0", + "Fixture": "v13.sql", + "ProvenanceSHA256": "9d9e23de5a05318f1f3625b0d375b64b10e42672ddd213f84a08c4375c0afdcc", + "Signature": "sha256:ef52be2fb56acd0e51b38506746fa3594ed1de76bb0b5dd180767d3c903fb46d", + "AppliedVersion": 13, + "HasSourceMetadata": false + }, + { + "Tag": "v2.13.0", + "Fixture": "v13.sql", + "ProvenanceSHA256": "9d9e23de5a05318f1f3625b0d375b64b10e42672ddd213f84a08c4375c0afdcc", + "Signature": "sha256:ef52be2fb56acd0e51b38506746fa3594ed1de76bb0b5dd180767d3c903fb46d", + "AppliedVersion": 13, + "HasSourceMetadata": false + }, + { + "Tag": "v2.14.0", + "Fixture": "v13.sql", + "ProvenanceSHA256": "9d9e23de5a05318f1f3625b0d375b64b10e42672ddd213f84a08c4375c0afdcc", + "Signature": "sha256:ef52be2fb56acd0e51b38506746fa3594ed1de76bb0b5dd180767d3c903fb46d", + "AppliedVersion": 13, + "HasSourceMetadata": false + }, + { + "Tag": "v2.14.1", + "Fixture": "v13.sql", + "ProvenanceSHA256": "9d9e23de5a05318f1f3625b0d375b64b10e42672ddd213f84a08c4375c0afdcc", + "Signature": "sha256:ef52be2fb56acd0e51b38506746fa3594ed1de76bb0b5dd180767d3c903fb46d", + "AppliedVersion": 13, + "HasSourceMetadata": false + }, + { + "Tag": "v2.14.2", + "Fixture": "v13.sql", + "ProvenanceSHA256": "9d9e23de5a05318f1f3625b0d375b64b10e42672ddd213f84a08c4375c0afdcc", + "Signature": "sha256:ef52be2fb56acd0e51b38506746fa3594ed1de76bb0b5dd180767d3c903fb46d", + "AppliedVersion": 13, + "HasSourceMetadata": false + }, + { + "Tag": "v2.14.3", + "Fixture": "v13.sql", + "ProvenanceSHA256": "9d9e23de5a05318f1f3625b0d375b64b10e42672ddd213f84a08c4375c0afdcc", + "Signature": "sha256:ef52be2fb56acd0e51b38506746fa3594ed1de76bb0b5dd180767d3c903fb46d", + "AppliedVersion": 13, + "HasSourceMetadata": false + }, + { + "Tag": "v2.15.0", + "Fixture": "v13.sql", + "ProvenanceSHA256": "9d9e23de5a05318f1f3625b0d375b64b10e42672ddd213f84a08c4375c0afdcc", + "Signature": "sha256:ef52be2fb56acd0e51b38506746fa3594ed1de76bb0b5dd180767d3c903fb46d", + "AppliedVersion": 13, + "HasSourceMetadata": false + }, + { + "Tag": "v2.15.1", + "Fixture": "v13.sql", + "ProvenanceSHA256": "9d9e23de5a05318f1f3625b0d375b64b10e42672ddd213f84a08c4375c0afdcc", + "Signature": "sha256:ef52be2fb56acd0e51b38506746fa3594ed1de76bb0b5dd180767d3c903fb46d", + "AppliedVersion": 13, + "HasSourceMetadata": false + }, + { + "Tag": "v2.16.0", + "Fixture": "v13.sql", + "ProvenanceSHA256": "9d9e23de5a05318f1f3625b0d375b64b10e42672ddd213f84a08c4375c0afdcc", + "Signature": "sha256:ef52be2fb56acd0e51b38506746fa3594ed1de76bb0b5dd180767d3c903fb46d", + "AppliedVersion": 13, + "HasSourceMetadata": false + }, + { + "Tag": "v2.16.1", + "Fixture": "v13.sql", + "ProvenanceSHA256": "9d9e23de5a05318f1f3625b0d375b64b10e42672ddd213f84a08c4375c0afdcc", + "Signature": "sha256:ef52be2fb56acd0e51b38506746fa3594ed1de76bb0b5dd180767d3c903fb46d", + "AppliedVersion": 13, + "HasSourceMetadata": false + }, + { + "Tag": "v3.0.0", + "Fixture": "v13.sql", + "ProvenanceSHA256": "9d9e23de5a05318f1f3625b0d375b64b10e42672ddd213f84a08c4375c0afdcc", + "Signature": "sha256:ef52be2fb56acd0e51b38506746fa3594ed1de76bb0b5dd180767d3c903fb46d", + "AppliedVersion": 13, + "HasSourceMetadata": false + }, + { + "Tag": "v3.0.1", + "Fixture": "v13.sql", + "ProvenanceSHA256": "9d9e23de5a05318f1f3625b0d375b64b10e42672ddd213f84a08c4375c0afdcc", + "Signature": "sha256:ef52be2fb56acd0e51b38506746fa3594ed1de76bb0b5dd180767d3c903fb46d", + "AppliedVersion": 13, + "HasSourceMetadata": false + }, + { + "Tag": "v3.0.2", + "Fixture": "v13.sql", + "ProvenanceSHA256": "9d9e23de5a05318f1f3625b0d375b64b10e42672ddd213f84a08c4375c0afdcc", + "Signature": "sha256:ef52be2fb56acd0e51b38506746fa3594ed1de76bb0b5dd180767d3c903fb46d", + "AppliedVersion": 13, + "HasSourceMetadata": false + }, + { + "Tag": "v3.1.0", + "Fixture": "v13.sql", + "ProvenanceSHA256": "9d9e23de5a05318f1f3625b0d375b64b10e42672ddd213f84a08c4375c0afdcc", + "Signature": "sha256:ef52be2fb56acd0e51b38506746fa3594ed1de76bb0b5dd180767d3c903fb46d", + "AppliedVersion": 13, + "HasSourceMetadata": false + }, + { + "Tag": "v3.2.0", + "Fixture": "v13.sql", + "ProvenanceSHA256": "9d9e23de5a05318f1f3625b0d375b64b10e42672ddd213f84a08c4375c0afdcc", + "Signature": "sha256:ef52be2fb56acd0e51b38506746fa3594ed1de76bb0b5dd180767d3c903fb46d", + "AppliedVersion": 13, + "HasSourceMetadata": false + }, + { + "Tag": "v3.2.1", + "Fixture": "v13.sql", + "ProvenanceSHA256": "9d9e23de5a05318f1f3625b0d375b64b10e42672ddd213f84a08c4375c0afdcc", + "Signature": "sha256:ef52be2fb56acd0e51b38506746fa3594ed1de76bb0b5dd180767d3c903fb46d", + "AppliedVersion": 13, + "HasSourceMetadata": false + }, + { + "Tag": "v3.2.2", + "Fixture": "v13.sql", + "ProvenanceSHA256": "9d9e23de5a05318f1f3625b0d375b64b10e42672ddd213f84a08c4375c0afdcc", + "Signature": "sha256:ef52be2fb56acd0e51b38506746fa3594ed1de76bb0b5dd180767d3c903fb46d", + "AppliedVersion": 13, + "HasSourceMetadata": false + }, + { + "Tag": "v3.2.3", + "Fixture": "v13.sql", + "ProvenanceSHA256": "9d9e23de5a05318f1f3625b0d375b64b10e42672ddd213f84a08c4375c0afdcc", + "Signature": "sha256:ef52be2fb56acd0e51b38506746fa3594ed1de76bb0b5dd180767d3c903fb46d", + "AppliedVersion": 13, + "HasSourceMetadata": false + }, + { + "Tag": "v3.2.4", + "Fixture": "v13.sql", + "ProvenanceSHA256": "9d9e23de5a05318f1f3625b0d375b64b10e42672ddd213f84a08c4375c0afdcc", + "Signature": "sha256:ef52be2fb56acd0e51b38506746fa3594ed1de76bb0b5dd180767d3c903fb46d", + "AppliedVersion": 13, + "HasSourceMetadata": false + }, + { + "Tag": "v3.2.5", + "Fixture": "v13.sql", + "ProvenanceSHA256": "9d9e23de5a05318f1f3625b0d375b64b10e42672ddd213f84a08c4375c0afdcc", + "Signature": "sha256:ef52be2fb56acd0e51b38506746fa3594ed1de76bb0b5dd180767d3c903fb46d", + "AppliedVersion": 13, + "HasSourceMetadata": false + } + ], + "UnmanifestedFixtures": [ + { + "Fixture": "v2.sql", + "ProvenanceSHA256": "9e5857f14e30fe1391445064bf924b0efb12fa81d0e5011de72ce5f880cd7f6a", + "Signature": "sha256:a4256a6029bbc953df37a12979fc81b1879ef9f8ba28ff5cfe6b78472dee804b", + "AppliedVersion": 2, + "HasSourceMetadata": true, + "Provenance": "No manifest release tag maps to this V2-only compatibility triangulation fixture." + }, + { + "Fixture": "v3-no-source-metadata.sql", + "ProvenanceSHA256": "42013f814ed39dbbcf8bc8e6465777d628a8d38694713f59caaeb30e89f28f68", + "Signature": "sha256:a7b05ddcb786f8d633fd2f27b19e0274fdf78dbd5c0e5ef420cecc831bcc3a8f", + "AppliedVersion": 3, + "HasSourceMetadata": false, + "Provenance": "No manifest release tag maps to this partially migrated V3 compatibility triangulation fixture." + }, + { + "Fixture": "v5-without-source-metadata.sql", + "ProvenanceSHA256": "fb705645b2f77017f1e9f512ba32246cefb9981cde76f78cd929f2659fdfee84", + "Signature": "sha256:95c1c0aa96f1093511dd4e159ec3b574aacdfc67136531b2fb9c3cd01e90aad0", + "AppliedVersion": 5, + "HasSourceMetadata": false, + "Provenance": "No manifest release tag maps to this partially migrated V5 compatibility triangulation fixture." + }, + { + "Fixture": "v6.sql", + "ProvenanceSHA256": "a42b23b541cc83d34ebd5571f0d7ef7771a540fe3d7e71b1a3e537a5e5932af0", + "Signature": "sha256:68f237fe4a85c97df36522ebdd54703afb97ea84c3f0b0cd45e6400c5e95e220", + "AppliedVersion": 6, + "HasSourceMetadata": false, + "Provenance": "No manifest release tag maps to this partial legacy per-step V6-before-V7 lineage fixture; no release shipped V6 alone." + }, + { + "Fixture": "v13-legacy-alter-built.sql", + "ProvenanceSHA256": "b2c09d4fb99893464e99a0d5b990ef84a23d7a46ecefc924ed5820eacc7b86ca", + "Signature": "sha256:19d65765b8b0d0bb7d1c41692712c395627e005b7aeccca5f887c75be781e314", + "AppliedVersion": 13, + "HasSourceMetadata": false, + "Provenance": "Explicit known legacy V13 shape reproduced from the ALTER-built current schema produced by the historical migration chain; supported despite conservative full-DDL signing." + }, + { + "Fixture": "v13-legacy-existing-schema-migrations.sql", + "ProvenanceSHA256": "75d9463f55287704fd861f0147b0b63a6f73c5252f55a561c848dbf09c86b6f0", + "Signature": "sha256:f52b4132322f3addb0fc01304f92ca6a2c2f4706e5ac010c59a5ea594cbd25b2", + "AppliedVersion": 13, + "HasSourceMetadata": false, + "Provenance": "Explicit known legacy V13 shape produced when compatibility migrations start from a database whose schema_migrations table predated the current SetupSchema formatting; supported despite conservative full-DDL signing." + } + ] +} diff --git a/internal/compat/testdata/release-schemas/v1.sql b/internal/compat/testdata/release-schemas/v1.sql new file mode 100644 index 0000000..10ec26d --- /dev/null +++ b/internal/compat/testdata/release-schemas/v1.sql @@ -0,0 +1,91 @@ +-- Backscroll release schema fixture: v1.sql +-- Hermetic schema-only fixture captured for compatibility tests. + +BEGIN TRANSACTION; +CREATE TABLE IF NOT EXISTS schema_migrations ( + version INTEGER PRIMARY KEY, + name TEXT NOT NULL, + applied_on TEXT NOT NULL, + checksum TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS indexed_files ( + path TEXT PRIMARY KEY, + hash TEXT NOT NULL, + last_indexed DATETIME DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS search_items ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source TEXT NOT NULL DEFAULT 'session', + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + role TEXT NOT NULL, + text TEXT NOT NULL, + timestamp TEXT, + uuid TEXT UNIQUE, + project TEXT, + content_type TEXT NOT NULL DEFAULT 'text', + source_metadata TEXT DEFAULT NULL +); + +CREATE INDEX IF NOT EXISTS idx_search_items_source_path ON search_items(source_path); +CREATE INDEX IF NOT EXISTS idx_search_items_project ON search_items(project); + +CREATE TABLE IF NOT EXISTS session_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + schema_version INTEGER NOT NULL DEFAULT 1, + source TEXT NOT NULL DEFAULT 'session', + source_path TEXT NOT NULL, + project TEXT, + ordinal INTEGER NOT NULL, + timestamp TEXT, + event_type TEXT NOT NULL, + actor TEXT, + role TEXT, + tool_name TEXT, + tool_id TEXT, + command TEXT, + cwd TEXT, + exit_code INTEGER, + is_error INTEGER, + snippet TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_session_events_order ON session_events(source_path, ordinal, timestamp, id); +CREATE INDEX IF NOT EXISTS idx_session_events_project ON session_events(project); + +CREATE TABLE IF NOT EXISTS session_tags ( + source_path TEXT NOT NULL, + tag TEXT NOT NULL, + PRIMARY KEY (source_path, tag) +); + +CREATE INDEX IF NOT EXISTS idx_session_tags_tag ON session_tags(tag); + +CREATE TABLE IF NOT EXISTS dynamic_stopwords (term TEXT PRIMARY KEY); + +CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5( + text, + content=search_items, + content_rowid=id, + tokenize='porter unicode61' +); + +CREATE VIRTUAL TABLE IF NOT EXISTS messages_vocab USING fts5vocab(messages_fts, 'row'); + +CREATE TRIGGER IF NOT EXISTS search_items_ai AFTER INSERT ON search_items BEGIN + INSERT INTO messages_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_ad AFTER DELETE ON search_items BEGIN + INSERT INTO messages_fts(messages_fts, rowid, text) VALUES('delete', old.id, old.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_au AFTER UPDATE ON search_items BEGIN + INSERT INTO messages_fts(messages_fts, rowid, text) VALUES('delete', old.id, old.text); + INSERT INTO messages_fts(rowid, text) VALUES (new.id, new.text); +END; + +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (1, 'V1 core schema', '1970-01-01 00:00:00', '4e07949ccd3912fb3c0e149be9a2e05fdd51f8cedb8df1f28b3bb5ac5afe532a'); +COMMIT; diff --git a/internal/compat/testdata/release-schemas/v10.sql b/internal/compat/testdata/release-schemas/v10.sql new file mode 100644 index 0000000..abc225d --- /dev/null +++ b/internal/compat/testdata/release-schemas/v10.sql @@ -0,0 +1,168 @@ +-- Backscroll release schema fixture: v10.sql +-- Hermetic schema-only fixture captured for compatibility tests. + +BEGIN TRANSACTION; +CREATE TABLE IF NOT EXISTS schema_migrations ( + version INTEGER PRIMARY KEY, + name TEXT NOT NULL, + applied_on TEXT NOT NULL, + checksum TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS indexed_files ( + path TEXT PRIMARY KEY, + hash TEXT NOT NULL, + last_indexed DATETIME DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS search_items ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source TEXT NOT NULL DEFAULT 'session', + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + role TEXT NOT NULL, + text TEXT NOT NULL, + timestamp TEXT, + uuid TEXT UNIQUE, + project TEXT, + content_type TEXT NOT NULL DEFAULT 'text', + extraction_version INTEGER, + was_interrupted INTEGER +); + +CREATE INDEX IF NOT EXISTS idx_search_items_source_path ON search_items(source_path); +CREATE INDEX IF NOT EXISTS idx_search_items_project ON search_items(project); + +CREATE TABLE IF NOT EXISTS session_tags ( + source_path TEXT NOT NULL, + tag TEXT NOT NULL, + PRIMARY KEY (source_path, tag) +); + +CREATE INDEX IF NOT EXISTS idx_session_tags_tag ON session_tags(tag); + +CREATE TABLE IF NOT EXISTS dynamic_stopwords (term TEXT PRIMARY KEY); + +CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5( + text, + content=search_items, + content_rowid=id, + tokenize='porter unicode61' +); + +CREATE VIRTUAL TABLE IF NOT EXISTS messages_vocab USING fts5vocab(messages_fts, 'row'); + +CREATE VIRTUAL TABLE IF NOT EXISTS tool_fts USING fts5( + text, + content=search_items, + content_rowid=id, + tokenize='trigram' +); + +CREATE VIRTUAL TABLE IF NOT EXISTS tool_vocab USING fts5vocab(tool_fts, 'row'); + +CREATE TRIGGER IF NOT EXISTS search_items_ai_tool AFTER INSERT ON search_items +WHEN new.content_type = 'tool' BEGIN + INSERT INTO tool_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_ai_msg AFTER INSERT ON search_items +WHEN new.content_type IN ('text', 'code', 'reasoning') BEGIN + INSERT INTO messages_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_ad_tool AFTER DELETE ON search_items +WHEN old.content_type = 'tool' BEGIN + INSERT INTO tool_fts(tool_fts, rowid, text) VALUES('delete', old.id, old.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_ad_msg AFTER DELETE ON search_items +WHEN old.content_type IN ('text', 'code', 'reasoning') BEGIN + INSERT INTO messages_fts(messages_fts, rowid, text) VALUES('delete', old.id, old.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_au_tool AFTER UPDATE ON search_items +WHEN old.content_type = 'tool' BEGIN + INSERT INTO tool_fts(tool_fts, rowid, text) VALUES('delete', old.id, old.text); + INSERT INTO tool_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_au_msg AFTER UPDATE ON search_items +WHEN old.content_type IN ('text', 'code', 'reasoning') BEGIN + INSERT INTO messages_fts(messages_fts, rowid, text) VALUES('delete', old.id, old.text); + INSERT INTO messages_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TABLE IF NOT EXISTS chunks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source_id TEXT NOT NULL, + chunk_idx INTEGER NOT NULL, + content TEXT NOT NULL, + token_count INTEGER NOT NULL, + created_at INTEGER NOT NULL, + embedding BLOB, + UNIQUE(source_id, chunk_idx) +); + +CREATE INDEX IF NOT EXISTS idx_chunks_source_id ON chunks (source_id); + +CREATE TABLE IF NOT EXISTS embedding_metadata ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + chunk_id INTEGER NOT NULL REFERENCES chunks(id) ON DELETE CASCADE, + model_name TEXT NOT NULL, + model_version TEXT NOT NULL, + dimensions INTEGER NOT NULL, + created_at INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS tool_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + message_uuid TEXT, + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + tool_name TEXT NOT NULL, + command_head TEXT, + is_error INTEGER, + exit_code INTEGER, + extraction_version INTEGER NOT NULL, + UNIQUE(source_path, ordinal) +); +CREATE INDEX IF NOT EXISTS idx_tool_events_tool ON tool_events(tool_name); +CREATE INDEX IF NOT EXISTS idx_tool_events_uuid ON tool_events(message_uuid); +CREATE UNIQUE INDEX IF NOT EXISTS idx_tool_events_uuid_unique ON tool_events(message_uuid) WHERE message_uuid IS NOT NULL; + +CREATE TABLE IF NOT EXISTS message_templates ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + signature TEXT UNIQUE NOT NULL, + normalization_version INTEGER NOT NULL, + template_text TEXT NOT NULL, + occurrence_count INTEGER NOT NULL DEFAULT 1, + first_seen TEXT, + last_seen TEXT +); +CREATE INDEX IF NOT EXISTS idx_templates_sig ON message_templates(signature); +CREATE INDEX IF NOT EXISTS idx_templates_version ON message_templates(normalization_version); + +CREATE TABLE IF NOT EXISTS template_matches ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + template_id INTEGER NOT NULL, + item_uuid TEXT, + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + UNIQUE(source_path, ordinal, template_id), + FOREIGN KEY(template_id) REFERENCES message_templates(id) +); +CREATE INDEX IF NOT EXISTS idx_matches_template ON template_matches(template_id); +CREATE INDEX IF NOT EXISTS idx_matches_uuid ON template_matches(item_uuid); + +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (1, 'V1 core schema', '1970-01-01 00:00:00', '4e07949ccd3912fb3c0e149be9a2e05fdd51f8cedb8df1f28b3bb5ac5afe532a'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (2, 'V2 embedding tables', '1970-01-01 00:00:00', '37dc9627f01f0e2d0fbea6bba5cd9f609d5da05089eeb9541a057dd2290cf8af'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (3, 'V3 embedding blob column', '1970-01-01 00:00:00', '36cd183f10ff84ab4753be027078cdd710b46efdc830053d4a641af725006ea5'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (4, 'V4 tool_fts trigram index', '1970-01-01 00:00:00', '77e59cf515f33c466282e0f3e921377f584794d0b7cade1704f566374a569a55'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (5, 'V5 drop phantom session_events', '1970-01-01 00:00:00', 'aedaab81efb6bc34d3f664468b71c1a716f5b55d5132156cb43bd7462d549c7b'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (6, 'V6 drop phantom source_metadata column', '1970-01-01 00:00:00', 'a327b9b6e7b8f5fe369c9fc08093ac80a87640daa515890af94b269d430e9378'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (7, 'V7 reasoning content_type routes to messages_fts', '1970-01-01 00:00:00', 'a80704442c2a0084f98e4bc53978364b14d1c6bd3bd99ba6119a2a9ecbd685e7'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (8, 'V8 perennity: extraction_version, was_interrupted, tool_events', '1970-01-01 00:00:00', '6853d72ded3bdc775b52507321c31df44bf35e8277719432ca8aa126ee16cec1'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (9, 'V9 tool_events uuid uniqueness index', '1970-01-01 00:00:00', 'b16094805a4e08f6e0dd56bce5266c7c5fd71934389da9d13c4132076e546ca2'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (10, 'V10 template mining: message_templates, template_matches', '1970-01-01 00:00:00', '0e548d0cb6c47147726f943bfc860500ca9a9bc821df0601998876ea5e9652c2'); +COMMIT; diff --git a/internal/compat/testdata/release-schemas/v11.sql b/internal/compat/testdata/release-schemas/v11.sql new file mode 100644 index 0000000..324566c --- /dev/null +++ b/internal/compat/testdata/release-schemas/v11.sql @@ -0,0 +1,182 @@ +-- Backscroll release schema fixture: v11.sql +-- Hermetic schema-only fixture captured for compatibility tests. + +BEGIN TRANSACTION; +CREATE TABLE IF NOT EXISTS schema_migrations ( + version INTEGER PRIMARY KEY, + name TEXT NOT NULL, + applied_on TEXT NOT NULL, + checksum TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS indexed_files ( + path TEXT PRIMARY KEY, + hash TEXT NOT NULL, + last_indexed DATETIME DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS search_items ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source TEXT NOT NULL DEFAULT 'session', + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + role TEXT NOT NULL, + text TEXT NOT NULL, + timestamp TEXT, + uuid TEXT UNIQUE, + project TEXT, + content_type TEXT NOT NULL DEFAULT 'text', + extraction_version INTEGER, + was_interrupted INTEGER +); + +CREATE INDEX IF NOT EXISTS idx_search_items_source_path ON search_items(source_path); +CREATE INDEX IF NOT EXISTS idx_search_items_project ON search_items(project); + +CREATE TABLE IF NOT EXISTS session_tags ( + source_path TEXT NOT NULL, + tag TEXT NOT NULL, + PRIMARY KEY (source_path, tag) +); + +CREATE INDEX IF NOT EXISTS idx_session_tags_tag ON session_tags(tag); + +CREATE TABLE IF NOT EXISTS dynamic_stopwords (term TEXT PRIMARY KEY); + +CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5( + text, + content=search_items, + content_rowid=id, + tokenize='porter unicode61' +); + +CREATE VIRTUAL TABLE IF NOT EXISTS messages_vocab USING fts5vocab(messages_fts, 'row'); + +CREATE VIRTUAL TABLE IF NOT EXISTS tool_fts USING fts5( + text, + content=search_items, + content_rowid=id, + tokenize='trigram' +); + +CREATE VIRTUAL TABLE IF NOT EXISTS tool_vocab USING fts5vocab(tool_fts, 'row'); + +CREATE TRIGGER IF NOT EXISTS search_items_ai_tool AFTER INSERT ON search_items +WHEN new.content_type = 'tool' BEGIN + INSERT INTO tool_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_ai_msg AFTER INSERT ON search_items +WHEN new.content_type IN ('text', 'code', 'reasoning') BEGIN + INSERT INTO messages_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_ad_tool AFTER DELETE ON search_items +WHEN old.content_type = 'tool' BEGIN + INSERT INTO tool_fts(tool_fts, rowid, text) VALUES('delete', old.id, old.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_ad_msg AFTER DELETE ON search_items +WHEN old.content_type IN ('text', 'code', 'reasoning') BEGIN + INSERT INTO messages_fts(messages_fts, rowid, text) VALUES('delete', old.id, old.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_au_tool AFTER UPDATE ON search_items +WHEN old.content_type = 'tool' BEGIN + INSERT INTO tool_fts(tool_fts, rowid, text) VALUES('delete', old.id, old.text); + INSERT INTO tool_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_au_msg AFTER UPDATE ON search_items +WHEN old.content_type IN ('text', 'code', 'reasoning') BEGIN + INSERT INTO messages_fts(messages_fts, rowid, text) VALUES('delete', old.id, old.text); + INSERT INTO messages_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TABLE IF NOT EXISTS chunks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source_id TEXT NOT NULL, + chunk_idx INTEGER NOT NULL, + content TEXT NOT NULL, + token_count INTEGER NOT NULL, + created_at INTEGER NOT NULL, + embedding BLOB, + UNIQUE(source_id, chunk_idx) +); + +CREATE INDEX IF NOT EXISTS idx_chunks_source_id ON chunks (source_id); + +CREATE TABLE IF NOT EXISTS embedding_metadata ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + chunk_id INTEGER NOT NULL REFERENCES chunks(id) ON DELETE CASCADE, + model_name TEXT NOT NULL, + model_version TEXT NOT NULL, + dimensions INTEGER NOT NULL, + created_at INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS tool_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + message_uuid TEXT, + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + tool_name TEXT NOT NULL, + command_head TEXT, + is_error INTEGER, + exit_code INTEGER, + extraction_version INTEGER NOT NULL, + UNIQUE(source_path, ordinal) +); +CREATE INDEX IF NOT EXISTS idx_tool_events_tool ON tool_events(tool_name); +CREATE INDEX IF NOT EXISTS idx_tool_events_uuid ON tool_events(message_uuid); +CREATE UNIQUE INDEX IF NOT EXISTS idx_tool_events_uuid_unique ON tool_events(message_uuid) WHERE message_uuid IS NOT NULL; + +CREATE TABLE IF NOT EXISTS message_templates ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + signature TEXT UNIQUE NOT NULL, + normalization_version INTEGER NOT NULL, + template_text TEXT NOT NULL, + occurrence_count INTEGER NOT NULL DEFAULT 1, + first_seen TEXT, + last_seen TEXT +); +CREATE INDEX IF NOT EXISTS idx_templates_sig ON message_templates(signature); +CREATE INDEX IF NOT EXISTS idx_templates_version ON message_templates(normalization_version); + +CREATE TABLE IF NOT EXISTS template_matches ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + template_id INTEGER NOT NULL, + item_uuid TEXT, + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + UNIQUE(source_path, ordinal, template_id), + FOREIGN KEY(template_id) REFERENCES message_templates(id) +); +CREATE INDEX IF NOT EXISTS idx_matches_template ON template_matches(template_id); +CREATE INDEX IF NOT EXISTS idx_matches_uuid ON template_matches(item_uuid); + +CREATE TABLE IF NOT EXISTS correction_signals ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + item_uuid TEXT, + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + detector TEXT NOT NULL, + confidence REAL NOT NULL, + extraction_version INTEGER NOT NULL, + UNIQUE(source_path, ordinal, detector) +); +CREATE INDEX IF NOT EXISTS idx_correction_signals_detector ON correction_signals(detector); +CREATE INDEX IF NOT EXISTS idx_correction_signals_confidence ON correction_signals(confidence DESC); + +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (1, 'V1 core schema', '1970-01-01 00:00:00', '4e07949ccd3912fb3c0e149be9a2e05fdd51f8cedb8df1f28b3bb5ac5afe532a'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (2, 'V2 embedding tables', '1970-01-01 00:00:00', '37dc9627f01f0e2d0fbea6bba5cd9f609d5da05089eeb9541a057dd2290cf8af'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (3, 'V3 embedding blob column', '1970-01-01 00:00:00', '36cd183f10ff84ab4753be027078cdd710b46efdc830053d4a641af725006ea5'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (4, 'V4 tool_fts trigram index', '1970-01-01 00:00:00', '77e59cf515f33c466282e0f3e921377f584794d0b7cade1704f566374a569a55'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (5, 'V5 drop phantom session_events', '1970-01-01 00:00:00', 'aedaab81efb6bc34d3f664468b71c1a716f5b55d5132156cb43bd7462d549c7b'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (6, 'V6 drop phantom source_metadata column', '1970-01-01 00:00:00', 'a327b9b6e7b8f5fe369c9fc08093ac80a87640daa515890af94b269d430e9378'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (7, 'V7 reasoning content_type routes to messages_fts', '1970-01-01 00:00:00', 'a80704442c2a0084f98e4bc53978364b14d1c6bd3bd99ba6119a2a9ecbd685e7'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (8, 'V8 perennity: extraction_version, was_interrupted, tool_events', '1970-01-01 00:00:00', '6853d72ded3bdc775b52507321c31df44bf35e8277719432ca8aa126ee16cec1'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (9, 'V9 tool_events uuid uniqueness index', '1970-01-01 00:00:00', 'b16094805a4e08f6e0dd56bce5266c7c5fd71934389da9d13c4132076e546ca2'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (10, 'V10 template mining: message_templates, template_matches', '1970-01-01 00:00:00', '0e548d0cb6c47147726f943bfc860500ca9a9bc821df0601998876ea5e9652c2'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (11, 'V11 correction detection: correction_signals', '1970-01-01 00:00:00', '5a7180f901c5feacb67cd104a61e7b7ba9cec69aaeab1d2b5d723459dc590456'); +COMMIT; diff --git a/internal/compat/testdata/release-schemas/v12.sql b/internal/compat/testdata/release-schemas/v12.sql new file mode 100644 index 0000000..ab2dcfc --- /dev/null +++ b/internal/compat/testdata/release-schemas/v12.sql @@ -0,0 +1,197 @@ +-- Backscroll release schema fixture: v12.sql +-- Hermetic schema-only fixture captured for compatibility tests. + +BEGIN TRANSACTION; +CREATE TABLE IF NOT EXISTS schema_migrations ( + version INTEGER PRIMARY KEY, + name TEXT NOT NULL, + applied_on TEXT NOT NULL, + checksum TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS indexed_files ( + path TEXT PRIMARY KEY, + hash TEXT NOT NULL, + last_indexed DATETIME DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS search_items ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source TEXT NOT NULL DEFAULT 'session', + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + role TEXT NOT NULL, + text TEXT NOT NULL, + timestamp TEXT, + uuid TEXT UNIQUE, + project TEXT, + content_type TEXT NOT NULL DEFAULT 'text', + extraction_version INTEGER, + was_interrupted INTEGER +); + +CREATE INDEX IF NOT EXISTS idx_search_items_source_path ON search_items(source_path); +CREATE INDEX IF NOT EXISTS idx_search_items_project ON search_items(project); + +CREATE TABLE IF NOT EXISTS session_tags ( + source_path TEXT NOT NULL, + tag TEXT NOT NULL, + PRIMARY KEY (source_path, tag) +); + +CREATE INDEX IF NOT EXISTS idx_session_tags_tag ON session_tags(tag); + +CREATE TABLE IF NOT EXISTS dynamic_stopwords (term TEXT PRIMARY KEY); + +CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5( + text, + content=search_items, + content_rowid=id, + tokenize='porter unicode61' +); + +CREATE VIRTUAL TABLE IF NOT EXISTS messages_vocab USING fts5vocab(messages_fts, 'row'); + +CREATE VIRTUAL TABLE IF NOT EXISTS tool_fts USING fts5( + text, + content=search_items, + content_rowid=id, + tokenize='trigram' +); + +CREATE VIRTUAL TABLE IF NOT EXISTS tool_vocab USING fts5vocab(tool_fts, 'row'); + +CREATE TRIGGER IF NOT EXISTS search_items_ai_tool AFTER INSERT ON search_items +WHEN new.content_type = 'tool' BEGIN + INSERT INTO tool_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_ai_msg AFTER INSERT ON search_items +WHEN new.content_type IN ('text', 'code', 'reasoning') BEGIN + INSERT INTO messages_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_ad_tool AFTER DELETE ON search_items +WHEN old.content_type = 'tool' BEGIN + INSERT INTO tool_fts(tool_fts, rowid, text) VALUES('delete', old.id, old.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_ad_msg AFTER DELETE ON search_items +WHEN old.content_type IN ('text', 'code', 'reasoning') BEGIN + INSERT INTO messages_fts(messages_fts, rowid, text) VALUES('delete', old.id, old.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_au_tool AFTER UPDATE ON search_items +WHEN old.content_type = 'tool' BEGIN + INSERT INTO tool_fts(tool_fts, rowid, text) VALUES('delete', old.id, old.text); + INSERT INTO tool_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_au_msg AFTER UPDATE ON search_items +WHEN old.content_type IN ('text', 'code', 'reasoning') BEGIN + INSERT INTO messages_fts(messages_fts, rowid, text) VALUES('delete', old.id, old.text); + INSERT INTO messages_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TABLE IF NOT EXISTS chunks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source_id TEXT NOT NULL, + chunk_idx INTEGER NOT NULL, + content TEXT NOT NULL, + token_count INTEGER NOT NULL, + created_at INTEGER NOT NULL, + embedding BLOB, + UNIQUE(source_id, chunk_idx) +); + +CREATE INDEX IF NOT EXISTS idx_chunks_source_id ON chunks (source_id); + +CREATE TABLE IF NOT EXISTS embedding_metadata ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + chunk_id INTEGER NOT NULL REFERENCES chunks(id) ON DELETE CASCADE, + model_name TEXT NOT NULL, + model_version TEXT NOT NULL, + dimensions INTEGER NOT NULL, + created_at INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS tool_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + message_uuid TEXT, + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + tool_name TEXT NOT NULL, + command_head TEXT, + is_error INTEGER, + exit_code INTEGER, + extraction_version INTEGER NOT NULL, + UNIQUE(source_path, ordinal) +); +CREATE INDEX IF NOT EXISTS idx_tool_events_tool ON tool_events(tool_name); +CREATE INDEX IF NOT EXISTS idx_tool_events_uuid ON tool_events(message_uuid); +CREATE UNIQUE INDEX IF NOT EXISTS idx_tool_events_uuid_unique ON tool_events(message_uuid) WHERE message_uuid IS NOT NULL; + +CREATE TABLE IF NOT EXISTS message_templates ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + signature TEXT UNIQUE NOT NULL, + normalization_version INTEGER NOT NULL, + template_text TEXT NOT NULL, + occurrence_count INTEGER NOT NULL DEFAULT 1, + first_seen TEXT, + last_seen TEXT +); +CREATE INDEX IF NOT EXISTS idx_templates_sig ON message_templates(signature); +CREATE INDEX IF NOT EXISTS idx_templates_version ON message_templates(normalization_version); + +CREATE TABLE IF NOT EXISTS template_matches ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + template_id INTEGER NOT NULL, + item_uuid TEXT, + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + UNIQUE(source_path, ordinal, template_id), + FOREIGN KEY(template_id) REFERENCES message_templates(id) +); +CREATE INDEX IF NOT EXISTS idx_matches_template ON template_matches(template_id); +CREATE INDEX IF NOT EXISTS idx_matches_uuid ON template_matches(item_uuid); + +CREATE TABLE IF NOT EXISTS correction_signals ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + item_uuid TEXT, + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + detector TEXT NOT NULL, + confidence REAL NOT NULL, + extraction_version INTEGER NOT NULL, + UNIQUE(source_path, ordinal, detector) +); +CREATE INDEX IF NOT EXISTS idx_correction_signals_detector ON correction_signals(detector); +CREATE INDEX IF NOT EXISTS idx_correction_signals_confidence ON correction_signals(confidence DESC); + +CREATE TABLE IF NOT EXISTS annotations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + item_uuid TEXT, + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + kind TEXT NOT NULL, + label TEXT NOT NULL, + source TEXT NOT NULL DEFAULT 'agent', + created_at TEXT NOT NULL, + UNIQUE(source_path, ordinal, kind) +); +CREATE INDEX IF NOT EXISTS idx_annotations_uuid ON annotations(item_uuid); +CREATE INDEX IF NOT EXISTS idx_annotations_kind ON annotations(kind); + +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (1, 'V1 core schema', '1970-01-01 00:00:00', '4e07949ccd3912fb3c0e149be9a2e05fdd51f8cedb8df1f28b3bb5ac5afe532a'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (2, 'V2 embedding tables', '1970-01-01 00:00:00', '37dc9627f01f0e2d0fbea6bba5cd9f609d5da05089eeb9541a057dd2290cf8af'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (3, 'V3 embedding blob column', '1970-01-01 00:00:00', '36cd183f10ff84ab4753be027078cdd710b46efdc830053d4a641af725006ea5'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (4, 'V4 tool_fts trigram index', '1970-01-01 00:00:00', '77e59cf515f33c466282e0f3e921377f584794d0b7cade1704f566374a569a55'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (5, 'V5 drop phantom session_events', '1970-01-01 00:00:00', 'aedaab81efb6bc34d3f664468b71c1a716f5b55d5132156cb43bd7462d549c7b'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (6, 'V6 drop phantom source_metadata column', '1970-01-01 00:00:00', 'a327b9b6e7b8f5fe369c9fc08093ac80a87640daa515890af94b269d430e9378'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (7, 'V7 reasoning content_type routes to messages_fts', '1970-01-01 00:00:00', 'a80704442c2a0084f98e4bc53978364b14d1c6bd3bd99ba6119a2a9ecbd685e7'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (8, 'V8 perennity: extraction_version, was_interrupted, tool_events', '1970-01-01 00:00:00', '6853d72ded3bdc775b52507321c31df44bf35e8277719432ca8aa126ee16cec1'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (9, 'V9 tool_events uuid uniqueness index', '1970-01-01 00:00:00', 'b16094805a4e08f6e0dd56bce5266c7c5fd71934389da9d13c4132076e546ca2'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (10, 'V10 template mining: message_templates, template_matches', '1970-01-01 00:00:00', '0e548d0cb6c47147726f943bfc860500ca9a9bc821df0601998876ea5e9652c2'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (11, 'V11 correction detection: correction_signals', '1970-01-01 00:00:00', '5a7180f901c5feacb67cd104a61e7b7ba9cec69aaeab1d2b5d723459dc590456'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (12, 'V12 agent classification: annotations (free-form labels; enum freeze deferred)', '1970-01-01 00:00:00', 'b3fb66fd2924a9f07a3e4ec0ba253fc1d6965664615a795be6b22d01ab292108'); +COMMIT; diff --git a/internal/compat/testdata/release-schemas/v13-legacy-alter-built.sql b/internal/compat/testdata/release-schemas/v13-legacy-alter-built.sql new file mode 100644 index 0000000..1a033b0 --- /dev/null +++ b/internal/compat/testdata/release-schemas/v13-legacy-alter-built.sql @@ -0,0 +1,211 @@ +BEGIN; + +CREATE TABLE annotations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + item_uuid TEXT, + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + kind TEXT NOT NULL, + label TEXT NOT NULL, + source TEXT NOT NULL DEFAULT 'agent', + created_at TEXT NOT NULL, + UNIQUE(source_path, ordinal, kind) +); + +CREATE TABLE chunks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source_id TEXT NOT NULL, + chunk_idx INTEGER NOT NULL, + content TEXT NOT NULL, + token_count INTEGER NOT NULL, + created_at INTEGER NOT NULL, embedding BLOB, + UNIQUE(source_id, chunk_idx) +); + +CREATE TABLE correction_signals ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + item_uuid TEXT, + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + detector TEXT NOT NULL, + confidence REAL NOT NULL, + extraction_version INTEGER NOT NULL, + UNIQUE(source_path, ordinal, detector) +); + +CREATE TABLE dynamic_stopwords (term TEXT PRIMARY KEY); + +CREATE TABLE embedding_metadata ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + chunk_id INTEGER NOT NULL REFERENCES chunks(id) ON DELETE CASCADE, + model_name TEXT NOT NULL, + model_version TEXT NOT NULL, + dimensions INTEGER NOT NULL, + created_at INTEGER NOT NULL +); + +CREATE TABLE indexed_files ( + path TEXT PRIMARY KEY, + hash TEXT NOT NULL, + last_indexed DATETIME DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE message_templates ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + signature TEXT UNIQUE NOT NULL, + normalization_version INTEGER NOT NULL, + template_text TEXT NOT NULL, + occurrence_count INTEGER NOT NULL DEFAULT 1, + first_seen TEXT, + last_seen TEXT +); + +CREATE VIRTUAL TABLE messages_fts USING fts5( + text, + content=search_items, + content_rowid=id, + tokenize='porter unicode61' +); + +CREATE VIRTUAL TABLE messages_vocab USING fts5vocab(messages_fts, 'row'); + +CREATE TABLE schema_migrations ( + version INTEGER PRIMARY KEY, + name TEXT NOT NULL, + applied_on TEXT NOT NULL, + checksum TEXT NOT NULL + ); + +CREATE TABLE search_items ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source TEXT NOT NULL DEFAULT 'session', + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + role TEXT NOT NULL, + text TEXT NOT NULL, + timestamp TEXT, + uuid TEXT UNIQUE, + project TEXT, + content_type TEXT NOT NULL DEFAULT 'text', + extraction_version INTEGER, + was_interrupted INTEGER +); + +CREATE TABLE session_tags ( + source_path TEXT NOT NULL, + tag TEXT NOT NULL, + PRIMARY KEY (source_path, tag) +); + +CREATE TABLE template_matches ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + template_id INTEGER NOT NULL, + item_uuid TEXT, + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + UNIQUE(source_path, ordinal, template_id), + FOREIGN KEY(template_id) REFERENCES message_templates(id) +); + +CREATE TABLE tool_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + message_uuid TEXT, + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + tool_name TEXT NOT NULL, + command_head TEXT, + is_error INTEGER, + exit_code INTEGER, + extraction_version INTEGER NOT NULL, + UNIQUE(source_path, ordinal) +); + +CREATE VIRTUAL TABLE tool_fts USING fts5( + text, + content=search_items, + content_rowid=id, + tokenize='trigram' +); + +CREATE VIRTUAL TABLE tool_vocab USING fts5vocab(tool_fts, 'row'); + +CREATE INDEX idx_annotations_kind ON annotations(kind); + +CREATE INDEX idx_annotations_uuid ON annotations(item_uuid); + +CREATE INDEX idx_chunks_source_id ON chunks (source_id); + +CREATE INDEX idx_correction_signals_confidence ON correction_signals(confidence DESC); + +CREATE INDEX idx_correction_signals_detector ON correction_signals(detector); + +CREATE INDEX idx_correction_signals_source ON correction_signals(source_path); + +CREATE INDEX idx_matches_template ON template_matches(template_id); + +CREATE INDEX idx_matches_uuid ON template_matches(item_uuid); + +CREATE INDEX idx_search_items_project ON search_items(project); + +CREATE INDEX idx_search_items_source_path ON search_items(source_path); + +CREATE INDEX idx_session_tags_tag ON session_tags(tag); + +CREATE INDEX idx_template_matches_source ON template_matches(source_path); + +CREATE INDEX idx_templates_sig ON message_templates(signature); + +CREATE INDEX idx_templates_version ON message_templates(normalization_version); + +CREATE INDEX idx_tool_events_tool ON tool_events(tool_name); + +CREATE INDEX idx_tool_events_uuid ON tool_events(message_uuid); + +CREATE UNIQUE INDEX idx_tool_events_uuid_unique ON tool_events(message_uuid) WHERE message_uuid IS NOT NULL; + +CREATE TRIGGER search_items_ad_msg AFTER DELETE ON search_items +WHEN old.content_type IN ('text', 'code', 'reasoning') BEGIN + INSERT INTO messages_fts(messages_fts, rowid, text) VALUES('delete', old.id, old.text); +END; + +CREATE TRIGGER search_items_ad_tool AFTER DELETE ON search_items +WHEN old.content_type = 'tool' BEGIN + INSERT INTO tool_fts(tool_fts, rowid, text) VALUES('delete', old.id, old.text); +END; + +CREATE TRIGGER search_items_ai_msg AFTER INSERT ON search_items +WHEN new.content_type IN ('text', 'code', 'reasoning') BEGIN + INSERT INTO messages_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TRIGGER search_items_ai_tool AFTER INSERT ON search_items +WHEN new.content_type = 'tool' BEGIN + INSERT INTO tool_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TRIGGER search_items_au_msg AFTER UPDATE ON search_items +WHEN old.content_type IN ('text', 'code', 'reasoning') BEGIN + INSERT INTO messages_fts(messages_fts, rowid, text) VALUES('delete', old.id, old.text); + INSERT INTO messages_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TRIGGER search_items_au_tool AFTER UPDATE ON search_items +WHEN old.content_type = 'tool' BEGIN + INSERT INTO tool_fts(tool_fts, rowid, text) VALUES('delete', old.id, old.text); + INSERT INTO tool_fts(rowid, text) VALUES (new.id, new.text); +END; + +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (1, 'V1 core schema', '1970-01-01 00:00:00', '4e07949ccd3912fb3c0e149be9a2e05fdd51f8cedb8df1f28b3bb5ac5afe532a'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (2, 'V2 embedding tables', '1970-01-01 00:00:00', '37dc9627f01f0e2d0fbea6bba5cd9f609d5da05089eeb9541a057dd2290cf8af'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (3, 'V3 embedding blob column', '1970-01-01 00:00:00', '36cd183f10ff84ab4753be027078cdd710b46efdc830053d4a641af725006ea5'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (4, 'V4 tool_fts trigram index', '1970-01-01 00:00:00', '77e59cf515f33c466282e0f3e921377f584794d0b7cade1704f566374a569a55'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (5, 'V5 drop phantom session_events', '1970-01-01 00:00:00', 'aedaab81efb6bc34d3f664468b71c1a716f5b55d5132156cb43bd7462d549c7b'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (6, 'V6 drop phantom source_metadata column', '1970-01-01 00:00:00', 'a327b9b6e7b8f5fe369c9fc08093ac80a87640daa515890af94b269d430e9378'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (7, 'V7 reasoning content_type routes to messages_fts', '1970-01-01 00:00:00', 'a80704442c2a0084f98e4bc53978364b14d1c6bd3bd99ba6119a2a9ecbd685e7'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (8, 'V8 perennity: extraction_version, was_interrupted, tool_events', '1970-01-01 00:00:00', '6853d72ded3bdc775b52507321c31df44bf35e8277719432ca8aa126ee16cec1'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (9, 'V9 tool_events uuid uniqueness index', '1970-01-01 00:00:00', 'b16094805a4e08f6e0dd56bce5266c7c5fd71934389da9d13c4132076e546ca2'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (10, 'V10 template mining: message_templates, template_matches', '1970-01-01 00:00:00', '0e548d0cb6c47147726f943bfc860500ca9a9bc821df0601998876ea5e9652c2'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (11, 'V11 correction detection: correction_signals', '1970-01-01 00:00:00', '5a7180f901c5feacb67cd104a61e7b7ba9cec69aaeab1d2b5d723459dc590456'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (12, 'V12 agent classification: annotations (free-form labels; enum freeze deferred)', '1970-01-01 00:00:00', 'b3fb66fd2924a9f07a3e4ec0ba253fc1d6965664615a795be6b22d01ab292108'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (13, 'V13 backfill discovery indexes', '1970-01-01 00:00:00', '2172ce531c670806933ffe3005fdc0a2ebb8eb3f84d2bd0d8fa608dddb5d136e'); +COMMIT; diff --git a/internal/compat/testdata/release-schemas/v13-legacy-existing-schema-migrations.sql b/internal/compat/testdata/release-schemas/v13-legacy-existing-schema-migrations.sql new file mode 100644 index 0000000..88361a9 --- /dev/null +++ b/internal/compat/testdata/release-schemas/v13-legacy-existing-schema-migrations.sql @@ -0,0 +1,211 @@ +BEGIN; + +CREATE TABLE annotations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + item_uuid TEXT, + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + kind TEXT NOT NULL, + label TEXT NOT NULL, + source TEXT NOT NULL DEFAULT 'agent', + created_at TEXT NOT NULL, + UNIQUE(source_path, ordinal, kind) +); + +CREATE TABLE chunks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source_id TEXT NOT NULL, + chunk_idx INTEGER NOT NULL, + content TEXT NOT NULL, + token_count INTEGER NOT NULL, + created_at INTEGER NOT NULL, embedding BLOB, + UNIQUE(source_id, chunk_idx) +); + +CREATE TABLE correction_signals ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + item_uuid TEXT, + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + detector TEXT NOT NULL, + confidence REAL NOT NULL, + extraction_version INTEGER NOT NULL, + UNIQUE(source_path, ordinal, detector) +); + +CREATE TABLE dynamic_stopwords (term TEXT PRIMARY KEY); + +CREATE TABLE embedding_metadata ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + chunk_id INTEGER NOT NULL REFERENCES chunks(id) ON DELETE CASCADE, + model_name TEXT NOT NULL, + model_version TEXT NOT NULL, + dimensions INTEGER NOT NULL, + created_at INTEGER NOT NULL +); + +CREATE TABLE indexed_files ( + path TEXT PRIMARY KEY, + hash TEXT NOT NULL, + last_indexed DATETIME DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE message_templates ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + signature TEXT UNIQUE NOT NULL, + normalization_version INTEGER NOT NULL, + template_text TEXT NOT NULL, + occurrence_count INTEGER NOT NULL DEFAULT 1, + first_seen TEXT, + last_seen TEXT +); + +CREATE VIRTUAL TABLE messages_fts USING fts5( + text, + content=search_items, + content_rowid=id, + tokenize='porter unicode61' +); + +CREATE VIRTUAL TABLE messages_vocab USING fts5vocab(messages_fts, 'row'); + +CREATE TABLE schema_migrations ( + version INTEGER PRIMARY KEY, + name TEXT NOT NULL, + applied_on TEXT NOT NULL, + checksum TEXT NOT NULL +); + +CREATE TABLE search_items ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source TEXT NOT NULL DEFAULT 'session', + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + role TEXT NOT NULL, + text TEXT NOT NULL, + timestamp TEXT, + uuid TEXT UNIQUE, + project TEXT, + content_type TEXT NOT NULL DEFAULT 'text', + extraction_version INTEGER, + was_interrupted INTEGER +); + +CREATE TABLE session_tags ( + source_path TEXT NOT NULL, + tag TEXT NOT NULL, + PRIMARY KEY (source_path, tag) +); + +CREATE TABLE template_matches ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + template_id INTEGER NOT NULL, + item_uuid TEXT, + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + UNIQUE(source_path, ordinal, template_id), + FOREIGN KEY(template_id) REFERENCES message_templates(id) +); + +CREATE TABLE tool_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + message_uuid TEXT, + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + tool_name TEXT NOT NULL, + command_head TEXT, + is_error INTEGER, + exit_code INTEGER, + extraction_version INTEGER NOT NULL, + UNIQUE(source_path, ordinal) +); + +CREATE VIRTUAL TABLE tool_fts USING fts5( + text, + content=search_items, + content_rowid=id, + tokenize='trigram' +); + +CREATE VIRTUAL TABLE tool_vocab USING fts5vocab(tool_fts, 'row'); + +CREATE INDEX idx_annotations_kind ON annotations(kind); + +CREATE INDEX idx_annotations_uuid ON annotations(item_uuid); + +CREATE INDEX idx_chunks_source_id ON chunks (source_id); + +CREATE INDEX idx_correction_signals_confidence ON correction_signals(confidence DESC); + +CREATE INDEX idx_correction_signals_detector ON correction_signals(detector); + +CREATE INDEX idx_correction_signals_source ON correction_signals(source_path); + +CREATE INDEX idx_matches_template ON template_matches(template_id); + +CREATE INDEX idx_matches_uuid ON template_matches(item_uuid); + +CREATE INDEX idx_search_items_project ON search_items(project); + +CREATE INDEX idx_search_items_source_path ON search_items(source_path); + +CREATE INDEX idx_session_tags_tag ON session_tags(tag); + +CREATE INDEX idx_template_matches_source ON template_matches(source_path); + +CREATE INDEX idx_templates_sig ON message_templates(signature); + +CREATE INDEX idx_templates_version ON message_templates(normalization_version); + +CREATE INDEX idx_tool_events_tool ON tool_events(tool_name); + +CREATE INDEX idx_tool_events_uuid ON tool_events(message_uuid); + +CREATE UNIQUE INDEX idx_tool_events_uuid_unique ON tool_events(message_uuid) WHERE message_uuid IS NOT NULL; + +CREATE TRIGGER search_items_ad_msg AFTER DELETE ON search_items +WHEN old.content_type IN ('text', 'code', 'reasoning') BEGIN + INSERT INTO messages_fts(messages_fts, rowid, text) VALUES('delete', old.id, old.text); +END; + +CREATE TRIGGER search_items_ad_tool AFTER DELETE ON search_items +WHEN old.content_type = 'tool' BEGIN + INSERT INTO tool_fts(tool_fts, rowid, text) VALUES('delete', old.id, old.text); +END; + +CREATE TRIGGER search_items_ai_msg AFTER INSERT ON search_items +WHEN new.content_type IN ('text', 'code', 'reasoning') BEGIN + INSERT INTO messages_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TRIGGER search_items_ai_tool AFTER INSERT ON search_items +WHEN new.content_type = 'tool' BEGIN + INSERT INTO tool_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TRIGGER search_items_au_msg AFTER UPDATE ON search_items +WHEN old.content_type IN ('text', 'code', 'reasoning') BEGIN + INSERT INTO messages_fts(messages_fts, rowid, text) VALUES('delete', old.id, old.text); + INSERT INTO messages_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TRIGGER search_items_au_tool AFTER UPDATE ON search_items +WHEN old.content_type = 'tool' BEGIN + INSERT INTO tool_fts(tool_fts, rowid, text) VALUES('delete', old.id, old.text); + INSERT INTO tool_fts(rowid, text) VALUES (new.id, new.text); +END; + +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (1, 'V1 core schema', '1970-01-01 00:00:00', '4e07949ccd3912fb3c0e149be9a2e05fdd51f8cedb8df1f28b3bb5ac5afe532a'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (2, 'V2 embedding tables', '1970-01-01 00:00:00', '37dc9627f01f0e2d0fbea6bba5cd9f609d5da05089eeb9541a057dd2290cf8af'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (3, 'V3 embedding blob column', '1970-01-01 00:00:00', '36cd183f10ff84ab4753be027078cdd710b46efdc830053d4a641af725006ea5'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (4, 'V4 tool_fts trigram index', '1970-01-01 00:00:00', '77e59cf515f33c466282e0f3e921377f584794d0b7cade1704f566374a569a55'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (5, 'V5 drop phantom session_events', '1970-01-01 00:00:00', 'aedaab81efb6bc34d3f664468b71c1a716f5b55d5132156cb43bd7462d549c7b'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (6, 'V6 drop phantom source_metadata column', '1970-01-01 00:00:00', 'a327b9b6e7b8f5fe369c9fc08093ac80a87640daa515890af94b269d430e9378'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (7, 'V7 reasoning content_type routes to messages_fts', '1970-01-01 00:00:00', 'a80704442c2a0084f98e4bc53978364b14d1c6bd3bd99ba6119a2a9ecbd685e7'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (8, 'V8 perennity: extraction_version, was_interrupted, tool_events', '1970-01-01 00:00:00', '6853d72ded3bdc775b52507321c31df44bf35e8277719432ca8aa126ee16cec1'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (9, 'V9 tool_events uuid uniqueness index', '1970-01-01 00:00:00', 'b16094805a4e08f6e0dd56bce5266c7c5fd71934389da9d13c4132076e546ca2'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (10, 'V10 template mining: message_templates, template_matches', '1970-01-01 00:00:00', '0e548d0cb6c47147726f943bfc860500ca9a9bc821df0601998876ea5e9652c2'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (11, 'V11 correction detection: correction_signals', '1970-01-01 00:00:00', '5a7180f901c5feacb67cd104a61e7b7ba9cec69aaeab1d2b5d723459dc590456'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (12, 'V12 agent classification: annotations (free-form labels; enum freeze deferred)', '1970-01-01 00:00:00', 'b3fb66fd2924a9f07a3e4ec0ba253fc1d6965664615a795be6b22d01ab292108'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (13, 'V13 backfill discovery indexes', '1970-01-01 00:00:00', '2172ce531c670806933ffe3005fdc0a2ebb8eb3f84d2bd0d8fa608dddb5d136e'); +COMMIT; diff --git a/internal/compat/testdata/release-schemas/v13.sql b/internal/compat/testdata/release-schemas/v13.sql new file mode 100644 index 0000000..7704716 --- /dev/null +++ b/internal/compat/testdata/release-schemas/v13.sql @@ -0,0 +1,201 @@ +-- Backscroll release schema fixture: v13.sql +-- Hermetic schema-only fixture captured for compatibility tests. + +BEGIN TRANSACTION; +CREATE TABLE IF NOT EXISTS schema_migrations ( + version INTEGER PRIMARY KEY, + name TEXT NOT NULL, + applied_on TEXT NOT NULL, + checksum TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS indexed_files ( + path TEXT PRIMARY KEY, + hash TEXT NOT NULL, + last_indexed DATETIME DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS search_items ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source TEXT NOT NULL DEFAULT 'session', + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + role TEXT NOT NULL, + text TEXT NOT NULL, + timestamp TEXT, + uuid TEXT UNIQUE, + project TEXT, + content_type TEXT NOT NULL DEFAULT 'text', + extraction_version INTEGER, + was_interrupted INTEGER +); + +CREATE INDEX IF NOT EXISTS idx_search_items_source_path ON search_items(source_path); +CREATE INDEX IF NOT EXISTS idx_search_items_project ON search_items(project); + +CREATE TABLE IF NOT EXISTS session_tags ( + source_path TEXT NOT NULL, + tag TEXT NOT NULL, + PRIMARY KEY (source_path, tag) +); + +CREATE INDEX IF NOT EXISTS idx_session_tags_tag ON session_tags(tag); + +CREATE TABLE IF NOT EXISTS dynamic_stopwords (term TEXT PRIMARY KEY); + +CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5( + text, + content=search_items, + content_rowid=id, + tokenize='porter unicode61' +); + +CREATE VIRTUAL TABLE IF NOT EXISTS messages_vocab USING fts5vocab(messages_fts, 'row'); + +CREATE VIRTUAL TABLE IF NOT EXISTS tool_fts USING fts5( + text, + content=search_items, + content_rowid=id, + tokenize='trigram' +); + +CREATE VIRTUAL TABLE IF NOT EXISTS tool_vocab USING fts5vocab(tool_fts, 'row'); + +CREATE TRIGGER IF NOT EXISTS search_items_ai_tool AFTER INSERT ON search_items +WHEN new.content_type = 'tool' BEGIN + INSERT INTO tool_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_ai_msg AFTER INSERT ON search_items +WHEN new.content_type IN ('text', 'code', 'reasoning') BEGIN + INSERT INTO messages_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_ad_tool AFTER DELETE ON search_items +WHEN old.content_type = 'tool' BEGIN + INSERT INTO tool_fts(tool_fts, rowid, text) VALUES('delete', old.id, old.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_ad_msg AFTER DELETE ON search_items +WHEN old.content_type IN ('text', 'code', 'reasoning') BEGIN + INSERT INTO messages_fts(messages_fts, rowid, text) VALUES('delete', old.id, old.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_au_tool AFTER UPDATE ON search_items +WHEN old.content_type = 'tool' BEGIN + INSERT INTO tool_fts(tool_fts, rowid, text) VALUES('delete', old.id, old.text); + INSERT INTO tool_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_au_msg AFTER UPDATE ON search_items +WHEN old.content_type IN ('text', 'code', 'reasoning') BEGIN + INSERT INTO messages_fts(messages_fts, rowid, text) VALUES('delete', old.id, old.text); + INSERT INTO messages_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TABLE IF NOT EXISTS chunks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source_id TEXT NOT NULL, + chunk_idx INTEGER NOT NULL, + content TEXT NOT NULL, + token_count INTEGER NOT NULL, + created_at INTEGER NOT NULL, + embedding BLOB, + UNIQUE(source_id, chunk_idx) +); + +CREATE INDEX IF NOT EXISTS idx_chunks_source_id ON chunks (source_id); + +CREATE TABLE IF NOT EXISTS embedding_metadata ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + chunk_id INTEGER NOT NULL REFERENCES chunks(id) ON DELETE CASCADE, + model_name TEXT NOT NULL, + model_version TEXT NOT NULL, + dimensions INTEGER NOT NULL, + created_at INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS tool_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + message_uuid TEXT, + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + tool_name TEXT NOT NULL, + command_head TEXT, + is_error INTEGER, + exit_code INTEGER, + extraction_version INTEGER NOT NULL, + UNIQUE(source_path, ordinal) +); +CREATE INDEX IF NOT EXISTS idx_tool_events_tool ON tool_events(tool_name); +CREATE INDEX IF NOT EXISTS idx_tool_events_uuid ON tool_events(message_uuid); +CREATE UNIQUE INDEX IF NOT EXISTS idx_tool_events_uuid_unique ON tool_events(message_uuid) WHERE message_uuid IS NOT NULL; + +CREATE TABLE IF NOT EXISTS message_templates ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + signature TEXT UNIQUE NOT NULL, + normalization_version INTEGER NOT NULL, + template_text TEXT NOT NULL, + occurrence_count INTEGER NOT NULL DEFAULT 1, + first_seen TEXT, + last_seen TEXT +); +CREATE INDEX IF NOT EXISTS idx_templates_sig ON message_templates(signature); +CREATE INDEX IF NOT EXISTS idx_templates_version ON message_templates(normalization_version); + +CREATE TABLE IF NOT EXISTS template_matches ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + template_id INTEGER NOT NULL, + item_uuid TEXT, + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + UNIQUE(source_path, ordinal, template_id), + FOREIGN KEY(template_id) REFERENCES message_templates(id) +); +CREATE INDEX IF NOT EXISTS idx_matches_template ON template_matches(template_id); +CREATE INDEX IF NOT EXISTS idx_matches_uuid ON template_matches(item_uuid); + +CREATE TABLE IF NOT EXISTS correction_signals ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + item_uuid TEXT, + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + detector TEXT NOT NULL, + confidence REAL NOT NULL, + extraction_version INTEGER NOT NULL, + UNIQUE(source_path, ordinal, detector) +); +CREATE INDEX IF NOT EXISTS idx_correction_signals_detector ON correction_signals(detector); +CREATE INDEX IF NOT EXISTS idx_correction_signals_confidence ON correction_signals(confidence DESC); + +CREATE TABLE IF NOT EXISTS annotations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + item_uuid TEXT, + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + kind TEXT NOT NULL, + label TEXT NOT NULL, + source TEXT NOT NULL DEFAULT 'agent', + created_at TEXT NOT NULL, + UNIQUE(source_path, ordinal, kind) +); +CREATE INDEX IF NOT EXISTS idx_annotations_uuid ON annotations(item_uuid); +CREATE INDEX IF NOT EXISTS idx_annotations_kind ON annotations(kind); + +CREATE INDEX IF NOT EXISTS idx_template_matches_source ON template_matches(source_path); +CREATE INDEX IF NOT EXISTS idx_correction_signals_source ON correction_signals(source_path); + +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (1, 'V1 core schema', '1970-01-01 00:00:00', '4e07949ccd3912fb3c0e149be9a2e05fdd51f8cedb8df1f28b3bb5ac5afe532a'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (2, 'V2 embedding tables', '1970-01-01 00:00:00', '37dc9627f01f0e2d0fbea6bba5cd9f609d5da05089eeb9541a057dd2290cf8af'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (3, 'V3 embedding blob column', '1970-01-01 00:00:00', '36cd183f10ff84ab4753be027078cdd710b46efdc830053d4a641af725006ea5'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (4, 'V4 tool_fts trigram index', '1970-01-01 00:00:00', '77e59cf515f33c466282e0f3e921377f584794d0b7cade1704f566374a569a55'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (5, 'V5 drop phantom session_events', '1970-01-01 00:00:00', 'aedaab81efb6bc34d3f664468b71c1a716f5b55d5132156cb43bd7462d549c7b'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (6, 'V6 drop phantom source_metadata column', '1970-01-01 00:00:00', 'a327b9b6e7b8f5fe369c9fc08093ac80a87640daa515890af94b269d430e9378'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (7, 'V7 reasoning content_type routes to messages_fts', '1970-01-01 00:00:00', 'a80704442c2a0084f98e4bc53978364b14d1c6bd3bd99ba6119a2a9ecbd685e7'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (8, 'V8 perennity: extraction_version, was_interrupted, tool_events', '1970-01-01 00:00:00', '6853d72ded3bdc775b52507321c31df44bf35e8277719432ca8aa126ee16cec1'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (9, 'V9 tool_events uuid uniqueness index', '1970-01-01 00:00:00', 'b16094805a4e08f6e0dd56bce5266c7c5fd71934389da9d13c4132076e546ca2'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (10, 'V10 template mining: message_templates, template_matches', '1970-01-01 00:00:00', '0e548d0cb6c47147726f943bfc860500ca9a9bc821df0601998876ea5e9652c2'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (11, 'V11 correction detection: correction_signals', '1970-01-01 00:00:00', '5a7180f901c5feacb67cd104a61e7b7ba9cec69aaeab1d2b5d723459dc590456'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (12, 'V12 agent classification: annotations (free-form labels; enum freeze deferred)', '1970-01-01 00:00:00', 'b3fb66fd2924a9f07a3e4ec0ba253fc1d6965664615a795be6b22d01ab292108'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (13, 'V13 backfill discovery indexes', '1970-01-01 00:00:00', '2172ce531c670806933ffe3005fdc0a2ebb8eb3f84d2bd0d8fa608dddb5d136e'); +COMMIT; diff --git a/internal/compat/testdata/release-schemas/v2.sql b/internal/compat/testdata/release-schemas/v2.sql new file mode 100644 index 0000000..ebb6cf1 --- /dev/null +++ b/internal/compat/testdata/release-schemas/v2.sql @@ -0,0 +1,117 @@ +-- Backscroll release schema fixture: v2.sql +-- Hermetic schema-only fixture captured for compatibility tests. +-- No manifest release tag maps to this fixture. +-- It preserves the V2 shape for compatibility triangulation only: V2 embedding +-- tables are present, but the published release inventory maps the affected +-- releases to the later V3 fixture shape. + +BEGIN TRANSACTION; +CREATE TABLE IF NOT EXISTS schema_migrations ( + version INTEGER PRIMARY KEY, + name TEXT NOT NULL, + applied_on TEXT NOT NULL, + checksum TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS indexed_files ( + path TEXT PRIMARY KEY, + hash TEXT NOT NULL, + last_indexed DATETIME DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS search_items ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source TEXT NOT NULL DEFAULT 'session', + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + role TEXT NOT NULL, + text TEXT NOT NULL, + timestamp TEXT, + uuid TEXT UNIQUE, + project TEXT, + content_type TEXT NOT NULL DEFAULT 'text', + source_metadata TEXT DEFAULT NULL +); + +CREATE INDEX IF NOT EXISTS idx_search_items_source_path ON search_items(source_path); +CREATE INDEX IF NOT EXISTS idx_search_items_project ON search_items(project); + +CREATE TABLE IF NOT EXISTS session_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + schema_version INTEGER NOT NULL DEFAULT 1, + source TEXT NOT NULL DEFAULT 'session', + source_path TEXT NOT NULL, + project TEXT, + ordinal INTEGER NOT NULL, + timestamp TEXT, + event_type TEXT NOT NULL, + actor TEXT, + role TEXT, + tool_name TEXT, + tool_id TEXT, + command TEXT, + cwd TEXT, + exit_code INTEGER, + is_error INTEGER, + snippet TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_session_events_order ON session_events(source_path, ordinal, timestamp, id); +CREATE INDEX IF NOT EXISTS idx_session_events_project ON session_events(project); + +CREATE TABLE IF NOT EXISTS session_tags ( + source_path TEXT NOT NULL, + tag TEXT NOT NULL, + PRIMARY KEY (source_path, tag) +); + +CREATE INDEX IF NOT EXISTS idx_session_tags_tag ON session_tags(tag); + +CREATE TABLE IF NOT EXISTS dynamic_stopwords (term TEXT PRIMARY KEY); + +CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5( + text, + content=search_items, + content_rowid=id, + tokenize='porter unicode61' +); + +CREATE VIRTUAL TABLE IF NOT EXISTS messages_vocab USING fts5vocab(messages_fts, 'row'); + +CREATE TRIGGER IF NOT EXISTS search_items_ai AFTER INSERT ON search_items BEGIN + INSERT INTO messages_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_ad AFTER DELETE ON search_items BEGIN + INSERT INTO messages_fts(messages_fts, rowid, text) VALUES('delete', old.id, old.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_au AFTER UPDATE ON search_items BEGIN + INSERT INTO messages_fts(messages_fts, rowid, text) VALUES('delete', old.id, old.text); + INSERT INTO messages_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TABLE IF NOT EXISTS chunks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source_id TEXT NOT NULL, + chunk_idx INTEGER NOT NULL, + content TEXT NOT NULL, + token_count INTEGER NOT NULL, + created_at INTEGER NOT NULL, + UNIQUE(source_id, chunk_idx) +); + +CREATE INDEX IF NOT EXISTS idx_chunks_source_id ON chunks (source_id); + +CREATE TABLE IF NOT EXISTS embedding_metadata ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + chunk_id INTEGER NOT NULL REFERENCES chunks(id) ON DELETE CASCADE, + model_name TEXT NOT NULL, + model_version TEXT NOT NULL, + dimensions INTEGER NOT NULL, + created_at INTEGER NOT NULL +); + +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (1, 'V1 core schema', '1970-01-01 00:00:00', '4e07949ccd3912fb3c0e149be9a2e05fdd51f8cedb8df1f28b3bb5ac5afe532a'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (2, 'V2 embedding tables', '1970-01-01 00:00:00', '37dc9627f01f0e2d0fbea6bba5cd9f609d5da05089eeb9541a057dd2290cf8af'); +COMMIT; diff --git a/internal/compat/testdata/release-schemas/v3-no-source-metadata.sql b/internal/compat/testdata/release-schemas/v3-no-source-metadata.sql new file mode 100644 index 0000000..d0904d8 --- /dev/null +++ b/internal/compat/testdata/release-schemas/v3-no-source-metadata.sql @@ -0,0 +1,118 @@ +-- Backscroll release schema fixture: v3-no-source-metadata.sql +-- Hermetic schema-only fixture captured for compatibility tests. +-- No manifest release tag maps to this fixture. +-- It preserves a partially migrated V3 shape for compatibility triangulation +-- only: source_metadata is absent from search_items even though no published +-- release in the manifest shipped this exact schema_migrations lineage. + +BEGIN TRANSACTION; +CREATE TABLE IF NOT EXISTS schema_migrations ( + version INTEGER PRIMARY KEY, + name TEXT NOT NULL, + applied_on TEXT NOT NULL, + checksum TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS indexed_files ( + path TEXT PRIMARY KEY, + hash TEXT NOT NULL, + last_indexed DATETIME DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS search_items ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source TEXT NOT NULL DEFAULT 'session', + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + role TEXT NOT NULL, + text TEXT NOT NULL, + timestamp TEXT, + uuid TEXT UNIQUE, + project TEXT, + content_type TEXT NOT NULL DEFAULT 'text' +); + +CREATE INDEX IF NOT EXISTS idx_search_items_source_path ON search_items(source_path); +CREATE INDEX IF NOT EXISTS idx_search_items_project ON search_items(project); + +CREATE TABLE IF NOT EXISTS session_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + schema_version INTEGER NOT NULL DEFAULT 1, + source TEXT NOT NULL DEFAULT 'session', + source_path TEXT NOT NULL, + project TEXT, + ordinal INTEGER NOT NULL, + timestamp TEXT, + event_type TEXT NOT NULL, + actor TEXT, + role TEXT, + tool_name TEXT, + tool_id TEXT, + command TEXT, + cwd TEXT, + exit_code INTEGER, + is_error INTEGER, + snippet TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_session_events_order ON session_events(source_path, ordinal, timestamp, id); +CREATE INDEX IF NOT EXISTS idx_session_events_project ON session_events(project); + +CREATE TABLE IF NOT EXISTS session_tags ( + source_path TEXT NOT NULL, + tag TEXT NOT NULL, + PRIMARY KEY (source_path, tag) +); + +CREATE INDEX IF NOT EXISTS idx_session_tags_tag ON session_tags(tag); + +CREATE TABLE IF NOT EXISTS dynamic_stopwords (term TEXT PRIMARY KEY); + +CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5( + text, + content=search_items, + content_rowid=id, + tokenize='porter unicode61' +); + +CREATE VIRTUAL TABLE IF NOT EXISTS messages_vocab USING fts5vocab(messages_fts, 'row'); + +CREATE TRIGGER IF NOT EXISTS search_items_ai AFTER INSERT ON search_items BEGIN + INSERT INTO messages_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_ad AFTER DELETE ON search_items BEGIN + INSERT INTO messages_fts(messages_fts, rowid, text) VALUES('delete', old.id, old.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_au AFTER UPDATE ON search_items BEGIN + INSERT INTO messages_fts(messages_fts, rowid, text) VALUES('delete', old.id, old.text); + INSERT INTO messages_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TABLE IF NOT EXISTS chunks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source_id TEXT NOT NULL, + chunk_idx INTEGER NOT NULL, + content TEXT NOT NULL, + token_count INTEGER NOT NULL, + created_at INTEGER NOT NULL, + embedding BLOB, + UNIQUE(source_id, chunk_idx) +); + +CREATE INDEX IF NOT EXISTS idx_chunks_source_id ON chunks (source_id); + +CREATE TABLE IF NOT EXISTS embedding_metadata ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + chunk_id INTEGER NOT NULL REFERENCES chunks(id) ON DELETE CASCADE, + model_name TEXT NOT NULL, + model_version TEXT NOT NULL, + dimensions INTEGER NOT NULL, + created_at INTEGER NOT NULL +); + +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (1, 'V1 core schema', '1970-01-01 00:00:00', '4e07949ccd3912fb3c0e149be9a2e05fdd51f8cedb8df1f28b3bb5ac5afe532a'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (2, 'V2 embedding tables', '1970-01-01 00:00:00', '37dc9627f01f0e2d0fbea6bba5cd9f609d5da05089eeb9541a057dd2290cf8af'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (3, 'V3 embedding blob column', '1970-01-01 00:00:00', '36cd183f10ff84ab4753be027078cdd710b46efdc830053d4a641af725006ea5'); +COMMIT; diff --git a/internal/compat/testdata/release-schemas/v3.sql b/internal/compat/testdata/release-schemas/v3.sql new file mode 100644 index 0000000..d85de6e --- /dev/null +++ b/internal/compat/testdata/release-schemas/v3.sql @@ -0,0 +1,115 @@ +-- Backscroll release schema fixture: v3.sql +-- Hermetic schema-only fixture captured for compatibility tests. + +BEGIN TRANSACTION; +CREATE TABLE IF NOT EXISTS schema_migrations ( + version INTEGER PRIMARY KEY, + name TEXT NOT NULL, + applied_on TEXT NOT NULL, + checksum TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS indexed_files ( + path TEXT PRIMARY KEY, + hash TEXT NOT NULL, + last_indexed DATETIME DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS search_items ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source TEXT NOT NULL DEFAULT 'session', + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + role TEXT NOT NULL, + text TEXT NOT NULL, + timestamp TEXT, + uuid TEXT UNIQUE, + project TEXT, + content_type TEXT NOT NULL DEFAULT 'text', + source_metadata TEXT DEFAULT NULL +); + +CREATE INDEX IF NOT EXISTS idx_search_items_source_path ON search_items(source_path); +CREATE INDEX IF NOT EXISTS idx_search_items_project ON search_items(project); + +CREATE TABLE IF NOT EXISTS session_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + schema_version INTEGER NOT NULL DEFAULT 1, + source TEXT NOT NULL DEFAULT 'session', + source_path TEXT NOT NULL, + project TEXT, + ordinal INTEGER NOT NULL, + timestamp TEXT, + event_type TEXT NOT NULL, + actor TEXT, + role TEXT, + tool_name TEXT, + tool_id TEXT, + command TEXT, + cwd TEXT, + exit_code INTEGER, + is_error INTEGER, + snippet TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_session_events_order ON session_events(source_path, ordinal, timestamp, id); +CREATE INDEX IF NOT EXISTS idx_session_events_project ON session_events(project); + +CREATE TABLE IF NOT EXISTS session_tags ( + source_path TEXT NOT NULL, + tag TEXT NOT NULL, + PRIMARY KEY (source_path, tag) +); + +CREATE INDEX IF NOT EXISTS idx_session_tags_tag ON session_tags(tag); + +CREATE TABLE IF NOT EXISTS dynamic_stopwords (term TEXT PRIMARY KEY); + +CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5( + text, + content=search_items, + content_rowid=id, + tokenize='porter unicode61' +); + +CREATE VIRTUAL TABLE IF NOT EXISTS messages_vocab USING fts5vocab(messages_fts, 'row'); + +CREATE TRIGGER IF NOT EXISTS search_items_ai AFTER INSERT ON search_items BEGIN + INSERT INTO messages_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_ad AFTER DELETE ON search_items BEGIN + INSERT INTO messages_fts(messages_fts, rowid, text) VALUES('delete', old.id, old.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_au AFTER UPDATE ON search_items BEGIN + INSERT INTO messages_fts(messages_fts, rowid, text) VALUES('delete', old.id, old.text); + INSERT INTO messages_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TABLE IF NOT EXISTS chunks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source_id TEXT NOT NULL, + chunk_idx INTEGER NOT NULL, + content TEXT NOT NULL, + token_count INTEGER NOT NULL, + created_at INTEGER NOT NULL, + embedding BLOB, + UNIQUE(source_id, chunk_idx) +); + +CREATE INDEX IF NOT EXISTS idx_chunks_source_id ON chunks (source_id); + +CREATE TABLE IF NOT EXISTS embedding_metadata ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + chunk_id INTEGER NOT NULL REFERENCES chunks(id) ON DELETE CASCADE, + model_name TEXT NOT NULL, + model_version TEXT NOT NULL, + dimensions INTEGER NOT NULL, + created_at INTEGER NOT NULL +); + +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (1, 'V1 core schema', '1970-01-01 00:00:00', '4e07949ccd3912fb3c0e149be9a2e05fdd51f8cedb8df1f28b3bb5ac5afe532a'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (2, 'V2 embedding tables', '1970-01-01 00:00:00', '37dc9627f01f0e2d0fbea6bba5cd9f609d5da05089eeb9541a057dd2290cf8af'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (3, 'V3 embedding blob column', '1970-01-01 00:00:00', '36cd183f10ff84ab4753be027078cdd710b46efdc830053d4a641af725006ea5'); +COMMIT; diff --git a/internal/compat/testdata/release-schemas/v4.sql b/internal/compat/testdata/release-schemas/v4.sql new file mode 100644 index 0000000..490f00b --- /dev/null +++ b/internal/compat/testdata/release-schemas/v4.sql @@ -0,0 +1,144 @@ +-- Backscroll release schema fixture: v4.sql +-- Hermetic schema-only fixture captured for compatibility tests. + +BEGIN TRANSACTION; +CREATE TABLE IF NOT EXISTS schema_migrations ( + version INTEGER PRIMARY KEY, + name TEXT NOT NULL, + applied_on TEXT NOT NULL, + checksum TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS indexed_files ( + path TEXT PRIMARY KEY, + hash TEXT NOT NULL, + last_indexed DATETIME DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS search_items ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source TEXT NOT NULL DEFAULT 'session', + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + role TEXT NOT NULL, + text TEXT NOT NULL, + timestamp TEXT, + uuid TEXT UNIQUE, + project TEXT, + content_type TEXT NOT NULL DEFAULT 'text', + source_metadata TEXT DEFAULT NULL +); + +CREATE INDEX IF NOT EXISTS idx_search_items_source_path ON search_items(source_path); +CREATE INDEX IF NOT EXISTS idx_search_items_project ON search_items(project); + +CREATE TABLE IF NOT EXISTS session_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + schema_version INTEGER NOT NULL DEFAULT 1, + source TEXT NOT NULL DEFAULT 'session', + source_path TEXT NOT NULL, + project TEXT, + ordinal INTEGER NOT NULL, + timestamp TEXT, + event_type TEXT NOT NULL, + actor TEXT, + role TEXT, + tool_name TEXT, + tool_id TEXT, + command TEXT, + cwd TEXT, + exit_code INTEGER, + is_error INTEGER, + snippet TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_session_events_order ON session_events(source_path, ordinal, timestamp, id); +CREATE INDEX IF NOT EXISTS idx_session_events_project ON session_events(project); + +CREATE TABLE IF NOT EXISTS session_tags ( + source_path TEXT NOT NULL, + tag TEXT NOT NULL, + PRIMARY KEY (source_path, tag) +); + +CREATE INDEX IF NOT EXISTS idx_session_tags_tag ON session_tags(tag); + +CREATE TABLE IF NOT EXISTS dynamic_stopwords (term TEXT PRIMARY KEY); + +CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5( + text, + content=search_items, + content_rowid=id, + tokenize='porter unicode61' +); + +CREATE VIRTUAL TABLE IF NOT EXISTS messages_vocab USING fts5vocab(messages_fts, 'row'); + +CREATE VIRTUAL TABLE IF NOT EXISTS tool_fts USING fts5( + text, + content=search_items, + content_rowid=id, + tokenize='trigram' +); + +CREATE VIRTUAL TABLE IF NOT EXISTS tool_vocab USING fts5vocab(tool_fts, 'row'); + +CREATE TRIGGER IF NOT EXISTS search_items_ai_tool AFTER INSERT ON search_items +WHEN new.content_type = 'tool' BEGIN + INSERT INTO tool_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_ai_msg AFTER INSERT ON search_items +WHEN new.content_type <> 'tool' BEGIN + INSERT INTO messages_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_ad_tool AFTER DELETE ON search_items +WHEN old.content_type = 'tool' BEGIN + INSERT INTO tool_fts(tool_fts, rowid, text) VALUES('delete', old.id, old.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_ad_msg AFTER DELETE ON search_items +WHEN old.content_type <> 'tool' BEGIN + INSERT INTO messages_fts(messages_fts, rowid, text) VALUES('delete', old.id, old.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_au_tool AFTER UPDATE ON search_items +WHEN old.content_type = 'tool' BEGIN + INSERT INTO tool_fts(tool_fts, rowid, text) VALUES('delete', old.id, old.text); + INSERT INTO tool_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_au_msg AFTER UPDATE ON search_items +WHEN old.content_type <> 'tool' BEGIN + INSERT INTO messages_fts(messages_fts, rowid, text) VALUES('delete', old.id, old.text); + INSERT INTO messages_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TABLE IF NOT EXISTS chunks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source_id TEXT NOT NULL, + chunk_idx INTEGER NOT NULL, + content TEXT NOT NULL, + token_count INTEGER NOT NULL, + created_at INTEGER NOT NULL, + embedding BLOB, + UNIQUE(source_id, chunk_idx) +); + +CREATE INDEX IF NOT EXISTS idx_chunks_source_id ON chunks (source_id); + +CREATE TABLE IF NOT EXISTS embedding_metadata ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + chunk_id INTEGER NOT NULL REFERENCES chunks(id) ON DELETE CASCADE, + model_name TEXT NOT NULL, + model_version TEXT NOT NULL, + dimensions INTEGER NOT NULL, + created_at INTEGER NOT NULL +); + +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (1, 'V1 core schema', '1970-01-01 00:00:00', '4e07949ccd3912fb3c0e149be9a2e05fdd51f8cedb8df1f28b3bb5ac5afe532a'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (2, 'V2 embedding tables', '1970-01-01 00:00:00', '37dc9627f01f0e2d0fbea6bba5cd9f609d5da05089eeb9541a057dd2290cf8af'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (3, 'V3 embedding blob column', '1970-01-01 00:00:00', '36cd183f10ff84ab4753be027078cdd710b46efdc830053d4a641af725006ea5'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (4, 'V4 tool_fts trigram index', '1970-01-01 00:00:00', '77e59cf515f33c466282e0f3e921377f584794d0b7cade1704f566374a569a55'); +COMMIT; diff --git a/internal/compat/testdata/release-schemas/v5-with-source-metadata.sql b/internal/compat/testdata/release-schemas/v5-with-source-metadata.sql new file mode 100644 index 0000000..d1a0ebd --- /dev/null +++ b/internal/compat/testdata/release-schemas/v5-with-source-metadata.sql @@ -0,0 +1,122 @@ +-- Backscroll release schema fixture: v5-with-source-metadata.sql +-- Hermetic schema-only fixture captured for compatibility tests. + +BEGIN TRANSACTION; +CREATE TABLE IF NOT EXISTS schema_migrations ( + version INTEGER PRIMARY KEY, + name TEXT NOT NULL, + applied_on TEXT NOT NULL, + checksum TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS indexed_files ( + path TEXT PRIMARY KEY, + hash TEXT NOT NULL, + last_indexed DATETIME DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS search_items ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source TEXT NOT NULL DEFAULT 'session', + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + role TEXT NOT NULL, + text TEXT NOT NULL, + timestamp TEXT, + uuid TEXT UNIQUE, + project TEXT, + content_type TEXT NOT NULL DEFAULT 'text', + source_metadata TEXT DEFAULT NULL +); + +CREATE INDEX IF NOT EXISTS idx_search_items_source_path ON search_items(source_path); +CREATE INDEX IF NOT EXISTS idx_search_items_project ON search_items(project); + +CREATE TABLE IF NOT EXISTS session_tags ( + source_path TEXT NOT NULL, + tag TEXT NOT NULL, + PRIMARY KEY (source_path, tag) +); + +CREATE INDEX IF NOT EXISTS idx_session_tags_tag ON session_tags(tag); + +CREATE TABLE IF NOT EXISTS dynamic_stopwords (term TEXT PRIMARY KEY); + +CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5( + text, + content=search_items, + content_rowid=id, + tokenize='porter unicode61' +); + +CREATE VIRTUAL TABLE IF NOT EXISTS messages_vocab USING fts5vocab(messages_fts, 'row'); + +CREATE VIRTUAL TABLE IF NOT EXISTS tool_fts USING fts5( + text, + content=search_items, + content_rowid=id, + tokenize='trigram' +); + +CREATE VIRTUAL TABLE IF NOT EXISTS tool_vocab USING fts5vocab(tool_fts, 'row'); + +CREATE TRIGGER IF NOT EXISTS search_items_ai_tool AFTER INSERT ON search_items +WHEN new.content_type = 'tool' BEGIN + INSERT INTO tool_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_ai_msg AFTER INSERT ON search_items +WHEN new.content_type <> 'tool' BEGIN + INSERT INTO messages_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_ad_tool AFTER DELETE ON search_items +WHEN old.content_type = 'tool' BEGIN + INSERT INTO tool_fts(tool_fts, rowid, text) VALUES('delete', old.id, old.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_ad_msg AFTER DELETE ON search_items +WHEN old.content_type <> 'tool' BEGIN + INSERT INTO messages_fts(messages_fts, rowid, text) VALUES('delete', old.id, old.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_au_tool AFTER UPDATE ON search_items +WHEN old.content_type = 'tool' BEGIN + INSERT INTO tool_fts(tool_fts, rowid, text) VALUES('delete', old.id, old.text); + INSERT INTO tool_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_au_msg AFTER UPDATE ON search_items +WHEN old.content_type <> 'tool' BEGIN + INSERT INTO messages_fts(messages_fts, rowid, text) VALUES('delete', old.id, old.text); + INSERT INTO messages_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TABLE IF NOT EXISTS chunks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source_id TEXT NOT NULL, + chunk_idx INTEGER NOT NULL, + content TEXT NOT NULL, + token_count INTEGER NOT NULL, + created_at INTEGER NOT NULL, + embedding BLOB, + UNIQUE(source_id, chunk_idx) +); + +CREATE INDEX IF NOT EXISTS idx_chunks_source_id ON chunks (source_id); + +CREATE TABLE IF NOT EXISTS embedding_metadata ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + chunk_id INTEGER NOT NULL REFERENCES chunks(id) ON DELETE CASCADE, + model_name TEXT NOT NULL, + model_version TEXT NOT NULL, + dimensions INTEGER NOT NULL, + created_at INTEGER NOT NULL +); + +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (1, 'V1 core schema', '1970-01-01 00:00:00', '4e07949ccd3912fb3c0e149be9a2e05fdd51f8cedb8df1f28b3bb5ac5afe532a'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (2, 'V2 embedding tables', '1970-01-01 00:00:00', '37dc9627f01f0e2d0fbea6bba5cd9f609d5da05089eeb9541a057dd2290cf8af'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (3, 'V3 embedding blob column', '1970-01-01 00:00:00', '36cd183f10ff84ab4753be027078cdd710b46efdc830053d4a641af725006ea5'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (4, 'V4 tool_fts trigram index', '1970-01-01 00:00:00', '77e59cf515f33c466282e0f3e921377f584794d0b7cade1704f566374a569a55'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (5, 'V5 drop phantom session_events', '1970-01-01 00:00:00', 'aedaab81efb6bc34d3f664468b71c1a716f5b55d5132156cb43bd7462d549c7b'); +COMMIT; diff --git a/internal/compat/testdata/release-schemas/v5-without-source-metadata.sql b/internal/compat/testdata/release-schemas/v5-without-source-metadata.sql new file mode 100644 index 0000000..47d1f45 --- /dev/null +++ b/internal/compat/testdata/release-schemas/v5-without-source-metadata.sql @@ -0,0 +1,125 @@ +-- Backscroll release schema fixture: v5-without-source-metadata.sql +-- Hermetic schema-only fixture captured for compatibility tests. +-- No manifest release tag maps to this fixture. +-- It preserves a partially migrated V5 shape for compatibility triangulation +-- only: session_events and source_metadata are absent together without claiming +-- a published release in the manifest shipped this exact shape. + +BEGIN TRANSACTION; +CREATE TABLE IF NOT EXISTS schema_migrations ( + version INTEGER PRIMARY KEY, + name TEXT NOT NULL, + applied_on TEXT NOT NULL, + checksum TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS indexed_files ( + path TEXT PRIMARY KEY, + hash TEXT NOT NULL, + last_indexed DATETIME DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS search_items ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source TEXT NOT NULL DEFAULT 'session', + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + role TEXT NOT NULL, + text TEXT NOT NULL, + timestamp TEXT, + uuid TEXT UNIQUE, + project TEXT, + content_type TEXT NOT NULL DEFAULT 'text' +); + +CREATE INDEX IF NOT EXISTS idx_search_items_source_path ON search_items(source_path); +CREATE INDEX IF NOT EXISTS idx_search_items_project ON search_items(project); + +CREATE TABLE IF NOT EXISTS session_tags ( + source_path TEXT NOT NULL, + tag TEXT NOT NULL, + PRIMARY KEY (source_path, tag) +); + +CREATE INDEX IF NOT EXISTS idx_session_tags_tag ON session_tags(tag); + +CREATE TABLE IF NOT EXISTS dynamic_stopwords (term TEXT PRIMARY KEY); + +CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5( + text, + content=search_items, + content_rowid=id, + tokenize='porter unicode61' +); + +CREATE VIRTUAL TABLE IF NOT EXISTS messages_vocab USING fts5vocab(messages_fts, 'row'); + +CREATE VIRTUAL TABLE IF NOT EXISTS tool_fts USING fts5( + text, + content=search_items, + content_rowid=id, + tokenize='trigram' +); + +CREATE VIRTUAL TABLE IF NOT EXISTS tool_vocab USING fts5vocab(tool_fts, 'row'); + +CREATE TRIGGER IF NOT EXISTS search_items_ai_tool AFTER INSERT ON search_items +WHEN new.content_type = 'tool' BEGIN + INSERT INTO tool_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_ai_msg AFTER INSERT ON search_items +WHEN new.content_type <> 'tool' BEGIN + INSERT INTO messages_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_ad_tool AFTER DELETE ON search_items +WHEN old.content_type = 'tool' BEGIN + INSERT INTO tool_fts(tool_fts, rowid, text) VALUES('delete', old.id, old.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_ad_msg AFTER DELETE ON search_items +WHEN old.content_type <> 'tool' BEGIN + INSERT INTO messages_fts(messages_fts, rowid, text) VALUES('delete', old.id, old.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_au_tool AFTER UPDATE ON search_items +WHEN old.content_type = 'tool' BEGIN + INSERT INTO tool_fts(tool_fts, rowid, text) VALUES('delete', old.id, old.text); + INSERT INTO tool_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_au_msg AFTER UPDATE ON search_items +WHEN old.content_type <> 'tool' BEGIN + INSERT INTO messages_fts(messages_fts, rowid, text) VALUES('delete', old.id, old.text); + INSERT INTO messages_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TABLE IF NOT EXISTS chunks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source_id TEXT NOT NULL, + chunk_idx INTEGER NOT NULL, + content TEXT NOT NULL, + token_count INTEGER NOT NULL, + created_at INTEGER NOT NULL, + embedding BLOB, + UNIQUE(source_id, chunk_idx) +); + +CREATE INDEX IF NOT EXISTS idx_chunks_source_id ON chunks (source_id); + +CREATE TABLE IF NOT EXISTS embedding_metadata ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + chunk_id INTEGER NOT NULL REFERENCES chunks(id) ON DELETE CASCADE, + model_name TEXT NOT NULL, + model_version TEXT NOT NULL, + dimensions INTEGER NOT NULL, + created_at INTEGER NOT NULL +); + +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (1, 'V1 core schema', '1970-01-01 00:00:00', '4e07949ccd3912fb3c0e149be9a2e05fdd51f8cedb8df1f28b3bb5ac5afe532a'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (2, 'V2 embedding tables', '1970-01-01 00:00:00', '37dc9627f01f0e2d0fbea6bba5cd9f609d5da05089eeb9541a057dd2290cf8af'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (3, 'V3 embedding blob column', '1970-01-01 00:00:00', '36cd183f10ff84ab4753be027078cdd710b46efdc830053d4a641af725006ea5'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (4, 'V4 tool_fts trigram index', '1970-01-01 00:00:00', '77e59cf515f33c466282e0f3e921377f584794d0b7cade1704f566374a569a55'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (5, 'V5 drop phantom session_events', '1970-01-01 00:00:00', 'aedaab81efb6bc34d3f664468b71c1a716f5b55d5132156cb43bd7462d549c7b'); +COMMIT; diff --git a/internal/compat/testdata/release-schemas/v6.sql b/internal/compat/testdata/release-schemas/v6.sql new file mode 100644 index 0000000..e9a9407 --- /dev/null +++ b/internal/compat/testdata/release-schemas/v6.sql @@ -0,0 +1,127 @@ +-- Backscroll release schema fixture: v6.sql +-- Hermetic schema-only fixture captured for compatibility tests. +-- No manifest release tag maps to this fixture. +-- It preserves a partial legacy per-step migration state after V6 and before +-- V7 for compatibility triangulation only: source_metadata has already been +-- dropped, while the reasoning trigger migration has not yet run. This does +-- not claim any published release shipped V6 alone. + +BEGIN TRANSACTION; +CREATE TABLE IF NOT EXISTS schema_migrations ( + version INTEGER PRIMARY KEY, + name TEXT NOT NULL, + applied_on TEXT NOT NULL, + checksum TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS indexed_files ( + path TEXT PRIMARY KEY, + hash TEXT NOT NULL, + last_indexed DATETIME DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS search_items ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source TEXT NOT NULL DEFAULT 'session', + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + role TEXT NOT NULL, + text TEXT NOT NULL, + timestamp TEXT, + uuid TEXT UNIQUE, + project TEXT, + content_type TEXT NOT NULL DEFAULT 'text' +); + +CREATE INDEX IF NOT EXISTS idx_search_items_source_path ON search_items(source_path); +CREATE INDEX IF NOT EXISTS idx_search_items_project ON search_items(project); + +CREATE TABLE IF NOT EXISTS session_tags ( + source_path TEXT NOT NULL, + tag TEXT NOT NULL, + PRIMARY KEY (source_path, tag) +); + +CREATE INDEX IF NOT EXISTS idx_session_tags_tag ON session_tags(tag); + +CREATE TABLE IF NOT EXISTS dynamic_stopwords (term TEXT PRIMARY KEY); + +CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5( + text, + content=search_items, + content_rowid=id, + tokenize='porter unicode61' +); + +CREATE VIRTUAL TABLE IF NOT EXISTS messages_vocab USING fts5vocab(messages_fts, 'row'); + +CREATE VIRTUAL TABLE IF NOT EXISTS tool_fts USING fts5( + text, + content=search_items, + content_rowid=id, + tokenize='trigram' +); + +CREATE VIRTUAL TABLE IF NOT EXISTS tool_vocab USING fts5vocab(tool_fts, 'row'); + +CREATE TRIGGER IF NOT EXISTS search_items_ai_tool AFTER INSERT ON search_items +WHEN new.content_type = 'tool' BEGIN + INSERT INTO tool_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_ai_msg AFTER INSERT ON search_items +WHEN new.content_type <> 'tool' BEGIN + INSERT INTO messages_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_ad_tool AFTER DELETE ON search_items +WHEN old.content_type = 'tool' BEGIN + INSERT INTO tool_fts(tool_fts, rowid, text) VALUES('delete', old.id, old.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_ad_msg AFTER DELETE ON search_items +WHEN old.content_type <> 'tool' BEGIN + INSERT INTO messages_fts(messages_fts, rowid, text) VALUES('delete', old.id, old.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_au_tool AFTER UPDATE ON search_items +WHEN old.content_type = 'tool' BEGIN + INSERT INTO tool_fts(tool_fts, rowid, text) VALUES('delete', old.id, old.text); + INSERT INTO tool_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_au_msg AFTER UPDATE ON search_items +WHEN old.content_type <> 'tool' BEGIN + INSERT INTO messages_fts(messages_fts, rowid, text) VALUES('delete', old.id, old.text); + INSERT INTO messages_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TABLE IF NOT EXISTS chunks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source_id TEXT NOT NULL, + chunk_idx INTEGER NOT NULL, + content TEXT NOT NULL, + token_count INTEGER NOT NULL, + created_at INTEGER NOT NULL, + embedding BLOB, + UNIQUE(source_id, chunk_idx) +); + +CREATE INDEX IF NOT EXISTS idx_chunks_source_id ON chunks (source_id); + +CREATE TABLE IF NOT EXISTS embedding_metadata ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + chunk_id INTEGER NOT NULL REFERENCES chunks(id) ON DELETE CASCADE, + model_name TEXT NOT NULL, + model_version TEXT NOT NULL, + dimensions INTEGER NOT NULL, + created_at INTEGER NOT NULL +); + +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (1, 'V1 core schema', '1970-01-01 00:00:00', '4e07949ccd3912fb3c0e149be9a2e05fdd51f8cedb8df1f28b3bb5ac5afe532a'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (2, 'V2 embedding tables', '1970-01-01 00:00:00', '37dc9627f01f0e2d0fbea6bba5cd9f609d5da05089eeb9541a057dd2290cf8af'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (3, 'V3 embedding blob column', '1970-01-01 00:00:00', '36cd183f10ff84ab4753be027078cdd710b46efdc830053d4a641af725006ea5'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (4, 'V4 tool_fts trigram index', '1970-01-01 00:00:00', '77e59cf515f33c466282e0f3e921377f584794d0b7cade1704f566374a569a55'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (5, 'V5 drop phantom session_events', '1970-01-01 00:00:00', 'aedaab81efb6bc34d3f664468b71c1a716f5b55d5132156cb43bd7462d549c7b'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (6, 'V6 drop phantom source_metadata column', '1970-01-01 00:00:00', 'a327b9b6e7b8f5fe369c9fc08093ac80a87640daa515890af94b269d430e9378'); +COMMIT; diff --git a/internal/compat/testdata/release-schemas/v7.sql b/internal/compat/testdata/release-schemas/v7.sql new file mode 100644 index 0000000..2e50402 --- /dev/null +++ b/internal/compat/testdata/release-schemas/v7.sql @@ -0,0 +1,123 @@ +-- Backscroll release schema fixture: v7.sql +-- Hermetic schema-only fixture captured for compatibility tests. + +BEGIN TRANSACTION; +CREATE TABLE IF NOT EXISTS schema_migrations ( + version INTEGER PRIMARY KEY, + name TEXT NOT NULL, + applied_on TEXT NOT NULL, + checksum TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS indexed_files ( + path TEXT PRIMARY KEY, + hash TEXT NOT NULL, + last_indexed DATETIME DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS search_items ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source TEXT NOT NULL DEFAULT 'session', + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + role TEXT NOT NULL, + text TEXT NOT NULL, + timestamp TEXT, + uuid TEXT UNIQUE, + project TEXT, + content_type TEXT NOT NULL DEFAULT 'text' +); + +CREATE INDEX IF NOT EXISTS idx_search_items_source_path ON search_items(source_path); +CREATE INDEX IF NOT EXISTS idx_search_items_project ON search_items(project); + +CREATE TABLE IF NOT EXISTS session_tags ( + source_path TEXT NOT NULL, + tag TEXT NOT NULL, + PRIMARY KEY (source_path, tag) +); + +CREATE INDEX IF NOT EXISTS idx_session_tags_tag ON session_tags(tag); + +CREATE TABLE IF NOT EXISTS dynamic_stopwords (term TEXT PRIMARY KEY); + +CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5( + text, + content=search_items, + content_rowid=id, + tokenize='porter unicode61' +); + +CREATE VIRTUAL TABLE IF NOT EXISTS messages_vocab USING fts5vocab(messages_fts, 'row'); + +CREATE VIRTUAL TABLE IF NOT EXISTS tool_fts USING fts5( + text, + content=search_items, + content_rowid=id, + tokenize='trigram' +); + +CREATE VIRTUAL TABLE IF NOT EXISTS tool_vocab USING fts5vocab(tool_fts, 'row'); + +CREATE TRIGGER IF NOT EXISTS search_items_ai_tool AFTER INSERT ON search_items +WHEN new.content_type = 'tool' BEGIN + INSERT INTO tool_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_ai_msg AFTER INSERT ON search_items +WHEN new.content_type IN ('text', 'code', 'reasoning') BEGIN + INSERT INTO messages_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_ad_tool AFTER DELETE ON search_items +WHEN old.content_type = 'tool' BEGIN + INSERT INTO tool_fts(tool_fts, rowid, text) VALUES('delete', old.id, old.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_ad_msg AFTER DELETE ON search_items +WHEN old.content_type IN ('text', 'code', 'reasoning') BEGIN + INSERT INTO messages_fts(messages_fts, rowid, text) VALUES('delete', old.id, old.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_au_tool AFTER UPDATE ON search_items +WHEN old.content_type = 'tool' BEGIN + INSERT INTO tool_fts(tool_fts, rowid, text) VALUES('delete', old.id, old.text); + INSERT INTO tool_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_au_msg AFTER UPDATE ON search_items +WHEN old.content_type IN ('text', 'code', 'reasoning') BEGIN + INSERT INTO messages_fts(messages_fts, rowid, text) VALUES('delete', old.id, old.text); + INSERT INTO messages_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TABLE IF NOT EXISTS chunks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source_id TEXT NOT NULL, + chunk_idx INTEGER NOT NULL, + content TEXT NOT NULL, + token_count INTEGER NOT NULL, + created_at INTEGER NOT NULL, + embedding BLOB, + UNIQUE(source_id, chunk_idx) +); + +CREATE INDEX IF NOT EXISTS idx_chunks_source_id ON chunks (source_id); + +CREATE TABLE IF NOT EXISTS embedding_metadata ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + chunk_id INTEGER NOT NULL REFERENCES chunks(id) ON DELETE CASCADE, + model_name TEXT NOT NULL, + model_version TEXT NOT NULL, + dimensions INTEGER NOT NULL, + created_at INTEGER NOT NULL +); + +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (1, 'V1 core schema', '1970-01-01 00:00:00', '4e07949ccd3912fb3c0e149be9a2e05fdd51f8cedb8df1f28b3bb5ac5afe532a'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (2, 'V2 embedding tables', '1970-01-01 00:00:00', '37dc9627f01f0e2d0fbea6bba5cd9f609d5da05089eeb9541a057dd2290cf8af'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (3, 'V3 embedding blob column', '1970-01-01 00:00:00', '36cd183f10ff84ab4753be027078cdd710b46efdc830053d4a641af725006ea5'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (4, 'V4 tool_fts trigram index', '1970-01-01 00:00:00', '77e59cf515f33c466282e0f3e921377f584794d0b7cade1704f566374a569a55'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (5, 'V5 drop phantom session_events', '1970-01-01 00:00:00', 'aedaab81efb6bc34d3f664468b71c1a716f5b55d5132156cb43bd7462d549c7b'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (6, 'V6 drop phantom source_metadata column', '1970-01-01 00:00:00', 'a327b9b6e7b8f5fe369c9fc08093ac80a87640daa515890af94b269d430e9378'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (7, 'V7 reasoning content_type routes to messages_fts', '1970-01-01 00:00:00', 'a80704442c2a0084f98e4bc53978364b14d1c6bd3bd99ba6119a2a9ecbd685e7'); +COMMIT; diff --git a/internal/compat/testdata/release-schemas/v8.sql b/internal/compat/testdata/release-schemas/v8.sql new file mode 100644 index 0000000..6e4cedd --- /dev/null +++ b/internal/compat/testdata/release-schemas/v8.sql @@ -0,0 +1,141 @@ +-- Backscroll release schema fixture: v8.sql +-- Hermetic schema-only fixture captured for compatibility tests. + +BEGIN TRANSACTION; +CREATE TABLE IF NOT EXISTS schema_migrations ( + version INTEGER PRIMARY KEY, + name TEXT NOT NULL, + applied_on TEXT NOT NULL, + checksum TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS indexed_files ( + path TEXT PRIMARY KEY, + hash TEXT NOT NULL, + last_indexed DATETIME DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS search_items ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source TEXT NOT NULL DEFAULT 'session', + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + role TEXT NOT NULL, + text TEXT NOT NULL, + timestamp TEXT, + uuid TEXT UNIQUE, + project TEXT, + content_type TEXT NOT NULL DEFAULT 'text', + extraction_version INTEGER, + was_interrupted INTEGER +); + +CREATE INDEX IF NOT EXISTS idx_search_items_source_path ON search_items(source_path); +CREATE INDEX IF NOT EXISTS idx_search_items_project ON search_items(project); + +CREATE TABLE IF NOT EXISTS session_tags ( + source_path TEXT NOT NULL, + tag TEXT NOT NULL, + PRIMARY KEY (source_path, tag) +); + +CREATE INDEX IF NOT EXISTS idx_session_tags_tag ON session_tags(tag); + +CREATE TABLE IF NOT EXISTS dynamic_stopwords (term TEXT PRIMARY KEY); + +CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5( + text, + content=search_items, + content_rowid=id, + tokenize='porter unicode61' +); + +CREATE VIRTUAL TABLE IF NOT EXISTS messages_vocab USING fts5vocab(messages_fts, 'row'); + +CREATE VIRTUAL TABLE IF NOT EXISTS tool_fts USING fts5( + text, + content=search_items, + content_rowid=id, + tokenize='trigram' +); + +CREATE VIRTUAL TABLE IF NOT EXISTS tool_vocab USING fts5vocab(tool_fts, 'row'); + +CREATE TRIGGER IF NOT EXISTS search_items_ai_tool AFTER INSERT ON search_items +WHEN new.content_type = 'tool' BEGIN + INSERT INTO tool_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_ai_msg AFTER INSERT ON search_items +WHEN new.content_type IN ('text', 'code', 'reasoning') BEGIN + INSERT INTO messages_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_ad_tool AFTER DELETE ON search_items +WHEN old.content_type = 'tool' BEGIN + INSERT INTO tool_fts(tool_fts, rowid, text) VALUES('delete', old.id, old.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_ad_msg AFTER DELETE ON search_items +WHEN old.content_type IN ('text', 'code', 'reasoning') BEGIN + INSERT INTO messages_fts(messages_fts, rowid, text) VALUES('delete', old.id, old.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_au_tool AFTER UPDATE ON search_items +WHEN old.content_type = 'tool' BEGIN + INSERT INTO tool_fts(tool_fts, rowid, text) VALUES('delete', old.id, old.text); + INSERT INTO tool_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_au_msg AFTER UPDATE ON search_items +WHEN old.content_type IN ('text', 'code', 'reasoning') BEGIN + INSERT INTO messages_fts(messages_fts, rowid, text) VALUES('delete', old.id, old.text); + INSERT INTO messages_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TABLE IF NOT EXISTS chunks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source_id TEXT NOT NULL, + chunk_idx INTEGER NOT NULL, + content TEXT NOT NULL, + token_count INTEGER NOT NULL, + created_at INTEGER NOT NULL, + embedding BLOB, + UNIQUE(source_id, chunk_idx) +); + +CREATE INDEX IF NOT EXISTS idx_chunks_source_id ON chunks (source_id); + +CREATE TABLE IF NOT EXISTS embedding_metadata ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + chunk_id INTEGER NOT NULL REFERENCES chunks(id) ON DELETE CASCADE, + model_name TEXT NOT NULL, + model_version TEXT NOT NULL, + dimensions INTEGER NOT NULL, + created_at INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS tool_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + message_uuid TEXT, + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + tool_name TEXT NOT NULL, + command_head TEXT, + is_error INTEGER, + exit_code INTEGER, + extraction_version INTEGER NOT NULL, + UNIQUE(source_path, ordinal) +); +CREATE INDEX IF NOT EXISTS idx_tool_events_tool ON tool_events(tool_name); +CREATE INDEX IF NOT EXISTS idx_tool_events_uuid ON tool_events(message_uuid); + +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (1, 'V1 core schema', '1970-01-01 00:00:00', '4e07949ccd3912fb3c0e149be9a2e05fdd51f8cedb8df1f28b3bb5ac5afe532a'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (2, 'V2 embedding tables', '1970-01-01 00:00:00', '37dc9627f01f0e2d0fbea6bba5cd9f609d5da05089eeb9541a057dd2290cf8af'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (3, 'V3 embedding blob column', '1970-01-01 00:00:00', '36cd183f10ff84ab4753be027078cdd710b46efdc830053d4a641af725006ea5'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (4, 'V4 tool_fts trigram index', '1970-01-01 00:00:00', '77e59cf515f33c466282e0f3e921377f584794d0b7cade1704f566374a569a55'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (5, 'V5 drop phantom session_events', '1970-01-01 00:00:00', 'aedaab81efb6bc34d3f664468b71c1a716f5b55d5132156cb43bd7462d549c7b'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (6, 'V6 drop phantom source_metadata column', '1970-01-01 00:00:00', 'a327b9b6e7b8f5fe369c9fc08093ac80a87640daa515890af94b269d430e9378'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (7, 'V7 reasoning content_type routes to messages_fts', '1970-01-01 00:00:00', 'a80704442c2a0084f98e4bc53978364b14d1c6bd3bd99ba6119a2a9ecbd685e7'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (8, 'V8 perennity: extraction_version, was_interrupted, tool_events', '1970-01-01 00:00:00', '6853d72ded3bdc775b52507321c31df44bf35e8277719432ca8aa126ee16cec1'); +COMMIT; diff --git a/internal/compat/testdata/release-schemas/v9.sql b/internal/compat/testdata/release-schemas/v9.sql new file mode 100644 index 0000000..1c2dca5 --- /dev/null +++ b/internal/compat/testdata/release-schemas/v9.sql @@ -0,0 +1,143 @@ +-- Backscroll release schema fixture: v9.sql +-- Hermetic schema-only fixture captured for compatibility tests. + +BEGIN TRANSACTION; +CREATE TABLE IF NOT EXISTS schema_migrations ( + version INTEGER PRIMARY KEY, + name TEXT NOT NULL, + applied_on TEXT NOT NULL, + checksum TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS indexed_files ( + path TEXT PRIMARY KEY, + hash TEXT NOT NULL, + last_indexed DATETIME DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS search_items ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source TEXT NOT NULL DEFAULT 'session', + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + role TEXT NOT NULL, + text TEXT NOT NULL, + timestamp TEXT, + uuid TEXT UNIQUE, + project TEXT, + content_type TEXT NOT NULL DEFAULT 'text', + extraction_version INTEGER, + was_interrupted INTEGER +); + +CREATE INDEX IF NOT EXISTS idx_search_items_source_path ON search_items(source_path); +CREATE INDEX IF NOT EXISTS idx_search_items_project ON search_items(project); + +CREATE TABLE IF NOT EXISTS session_tags ( + source_path TEXT NOT NULL, + tag TEXT NOT NULL, + PRIMARY KEY (source_path, tag) +); + +CREATE INDEX IF NOT EXISTS idx_session_tags_tag ON session_tags(tag); + +CREATE TABLE IF NOT EXISTS dynamic_stopwords (term TEXT PRIMARY KEY); + +CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5( + text, + content=search_items, + content_rowid=id, + tokenize='porter unicode61' +); + +CREATE VIRTUAL TABLE IF NOT EXISTS messages_vocab USING fts5vocab(messages_fts, 'row'); + +CREATE VIRTUAL TABLE IF NOT EXISTS tool_fts USING fts5( + text, + content=search_items, + content_rowid=id, + tokenize='trigram' +); + +CREATE VIRTUAL TABLE IF NOT EXISTS tool_vocab USING fts5vocab(tool_fts, 'row'); + +CREATE TRIGGER IF NOT EXISTS search_items_ai_tool AFTER INSERT ON search_items +WHEN new.content_type = 'tool' BEGIN + INSERT INTO tool_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_ai_msg AFTER INSERT ON search_items +WHEN new.content_type IN ('text', 'code', 'reasoning') BEGIN + INSERT INTO messages_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_ad_tool AFTER DELETE ON search_items +WHEN old.content_type = 'tool' BEGIN + INSERT INTO tool_fts(tool_fts, rowid, text) VALUES('delete', old.id, old.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_ad_msg AFTER DELETE ON search_items +WHEN old.content_type IN ('text', 'code', 'reasoning') BEGIN + INSERT INTO messages_fts(messages_fts, rowid, text) VALUES('delete', old.id, old.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_au_tool AFTER UPDATE ON search_items +WHEN old.content_type = 'tool' BEGIN + INSERT INTO tool_fts(tool_fts, rowid, text) VALUES('delete', old.id, old.text); + INSERT INTO tool_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_au_msg AFTER UPDATE ON search_items +WHEN old.content_type IN ('text', 'code', 'reasoning') BEGIN + INSERT INTO messages_fts(messages_fts, rowid, text) VALUES('delete', old.id, old.text); + INSERT INTO messages_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TABLE IF NOT EXISTS chunks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source_id TEXT NOT NULL, + chunk_idx INTEGER NOT NULL, + content TEXT NOT NULL, + token_count INTEGER NOT NULL, + created_at INTEGER NOT NULL, + embedding BLOB, + UNIQUE(source_id, chunk_idx) +); + +CREATE INDEX IF NOT EXISTS idx_chunks_source_id ON chunks (source_id); + +CREATE TABLE IF NOT EXISTS embedding_metadata ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + chunk_id INTEGER NOT NULL REFERENCES chunks(id) ON DELETE CASCADE, + model_name TEXT NOT NULL, + model_version TEXT NOT NULL, + dimensions INTEGER NOT NULL, + created_at INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS tool_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + message_uuid TEXT, + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + tool_name TEXT NOT NULL, + command_head TEXT, + is_error INTEGER, + exit_code INTEGER, + extraction_version INTEGER NOT NULL, + UNIQUE(source_path, ordinal) +); +CREATE INDEX IF NOT EXISTS idx_tool_events_tool ON tool_events(tool_name); +CREATE INDEX IF NOT EXISTS idx_tool_events_uuid ON tool_events(message_uuid); +CREATE UNIQUE INDEX IF NOT EXISTS idx_tool_events_uuid_unique ON tool_events(message_uuid) WHERE message_uuid IS NOT NULL; + +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (1, 'V1 core schema', '1970-01-01 00:00:00', '4e07949ccd3912fb3c0e149be9a2e05fdd51f8cedb8df1f28b3bb5ac5afe532a'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (2, 'V2 embedding tables', '1970-01-01 00:00:00', '37dc9627f01f0e2d0fbea6bba5cd9f609d5da05089eeb9541a057dd2290cf8af'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (3, 'V3 embedding blob column', '1970-01-01 00:00:00', '36cd183f10ff84ab4753be027078cdd710b46efdc830053d4a641af725006ea5'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (4, 'V4 tool_fts trigram index', '1970-01-01 00:00:00', '77e59cf515f33c466282e0f3e921377f584794d0b7cade1704f566374a569a55'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (5, 'V5 drop phantom session_events', '1970-01-01 00:00:00', 'aedaab81efb6bc34d3f664468b71c1a716f5b55d5132156cb43bd7462d549c7b'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (6, 'V6 drop phantom source_metadata column', '1970-01-01 00:00:00', 'a327b9b6e7b8f5fe369c9fc08093ac80a87640daa515890af94b269d430e9378'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (7, 'V7 reasoning content_type routes to messages_fts', '1970-01-01 00:00:00', 'a80704442c2a0084f98e4bc53978364b14d1c6bd3bd99ba6119a2a9ecbd685e7'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (8, 'V8 perennity: extraction_version, was_interrupted, tool_events', '1970-01-01 00:00:00', '6853d72ded3bdc775b52507321c31df44bf35e8277719432ca8aa126ee16cec1'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (9, 'V9 tool_events uuid uniqueness index', '1970-01-01 00:00:00', 'b16094805a4e08f6e0dd56bce5266c7c5fd71934389da9d13c4132076e546ca2'); +COMMIT; diff --git a/internal/compat/types.go b/internal/compat/types.go new file mode 100644 index 0000000..7dc1229 --- /dev/null +++ b/internal/compat/types.go @@ -0,0 +1,61 @@ +package compat + +import ( + "context" + "database/sql" + + "github.com/pablontiv/backscroll/internal/models" +) + +type Code string + +const ( + CodeUnsupportedLineage Code = "unsupported_lineage" + CodeMigrationFailed Code = "migration_failed" + CodeIndexStale Code = "index_stale" + CodeRecoveryConflict Code = "recovery_conflict" + CodeUninterpretableRow Code = "uninterpretable_row" +) + +type Diagnostic struct { + Code Code + Summary string + Continuation []string +} + +type SchemaShape struct { + AppliedVersion int + Signature string +} + +type MigrationStep struct { + Version int + Name string +} + +type MigrationPlan struct { + From SchemaShape + Steps []MigrationStep +} + +type RecoveryInput struct { + Shape SchemaShape + Records []models.IndexedRecord + RowCount int +} + +type CanonicalRecord struct { + Record models.IndexedRecord + PayloadHash string +} + +type RecoveryPlan struct { + InputShapes []SchemaShape + Records []CanonicalRecord + ExactDuplicates int +} + +type Queryer interface { + QueryContext(context.Context, string, ...any) (*sql.Rows, error) + QueryRowContext(context.Context, string, ...any) *sql.Row +} diff --git a/internal/models/indexed_record.go b/internal/models/indexed_record.go new file mode 100644 index 0000000..2e17037 --- /dev/null +++ b/internal/models/indexed_record.go @@ -0,0 +1,14 @@ +package models + +// IndexedRecord represents a single canonical record from search_items. +type IndexedRecord struct { + Source string + SourcePath string + Ordinal int64 + Role string + Text string + Project *string + UUID *string + Timestamp *string + ContentType string +} diff --git a/internal/recovery/no_clobber_darwin.go b/internal/recovery/no_clobber_darwin.go new file mode 100644 index 0000000..14bc194 --- /dev/null +++ b/internal/recovery/no_clobber_darwin.go @@ -0,0 +1,9 @@ +//go:build darwin + +package recovery + +import "golang.org/x/sys/unix" + +func noClobberMove(oldPath, newPath string) error { + return unix.RenameatxNp(unix.AT_FDCWD, oldPath, unix.AT_FDCWD, newPath, unix.RENAME_EXCL) +} diff --git a/internal/recovery/no_clobber_fallback.go b/internal/recovery/no_clobber_fallback.go new file mode 100644 index 0000000..0069cd0 --- /dev/null +++ b/internal/recovery/no_clobber_fallback.go @@ -0,0 +1,7 @@ +//go:build !darwin && !linux + +package recovery + +func noClobberMove(oldPath, newPath string) error { + return noClobberUnsupportedError{oldPath: oldPath, newPath: newPath} +} diff --git a/internal/recovery/no_clobber_linux.go b/internal/recovery/no_clobber_linux.go new file mode 100644 index 0000000..ca74ad5 --- /dev/null +++ b/internal/recovery/no_clobber_linux.go @@ -0,0 +1,13 @@ +//go:build linux + +package recovery + +import "golang.org/x/sys/unix" + +func noClobberMove(oldPath, newPath string) error { + err := unix.Renameat2(unix.AT_FDCWD, oldPath, unix.AT_FDCWD, newPath, unix.RENAME_NOREPLACE) + if err == unix.ENOSYS || err == unix.EINVAL { + return noClobberUnsupportedError{oldPath: oldPath, newPath: newPath, cause: err} + } + return err +} diff --git a/internal/recovery/recovery.go b/internal/recovery/recovery.go new file mode 100644 index 0000000..4a3a043 --- /dev/null +++ b/internal/recovery/recovery.go @@ -0,0 +1,1275 @@ +package recovery + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "database/sql" + "encoding/hex" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" + + "github.com/pablontiv/backscroll/internal/compat" + "github.com/pablontiv/backscroll/internal/storage" +) + +// Options configures stranded database recovery planning. +type Options struct { + ActivePath string + FromPath string + DryRun bool +} + +// Report describes the recovery plan without embedding writable state. +type Report struct { + ActivePath string + BackupPath string + InputCounts []int + ExactDuplicates int + FinalCount int + Shapes []compat.SchemaShape + Conflicts []compat.Diagnostic +} + +type resolvedPath struct { + path string + info os.FileInfo +} + +type recoveryOperationDeps struct { + createDestination func(context.Context, string, compat.RecoveryPlan) (string, error) + verifyDestination func(context.Context, string, compat.RecoveryPlan) error + noClobberMove func(string, string) error + installMove func(string, string) error + restoreMove func(string, string) error + syncDir func(string) error + syncFile func(string) error + randomHex func(int) (string, error) + fingerprintSet func(string) (sqliteFileSetSnapshot, error) + ensureSidecars func(string) error + openReservation func(context.Context, string) (*activeReservation, error) + readReservedInput func(context.Context, compat.Queryer) (compat.RecoveryInput, *compat.Diagnostic, error) + closeReservation func(*activeReservation) error + cleanupFileSet func(string) error +} + +func defaultRecoveryOperationDeps() recoveryOperationDeps { + return recoveryOperationDeps{ + createDestination: storage.CreateRecoveryDestination, + verifyDestination: storage.VerifyRecoveryDestination, + noClobberMove: noClobberMove, + installMove: noClobberMove, + restoreMove: noClobberMove, + syncDir: syncDir, + syncFile: syncFile, + randomHex: randomHex, + fingerprintSet: snapshotSQLiteFileSet, + ensureSidecars: ensureSourceSidecarsSafe, + openReservation: openActiveReservation, + readReservedInput: storage.ReadRecoveryInputFromQueryer, + closeReservation: func(r *activeReservation) error { return r.Close() }, + cleanupFileSet: removeSQLiteFileSet, + } +} + +// Execute resolves active and stranded database paths, reads distinct inputs in +// immutable read-only mode, runs the deterministic compatibility recovery planner, +// and optionally installs a verified fresh destination with a preserved backup. +func Execute(ctx context.Context, opts Options) (report Report, err error) { + return executeWithDeps(ctx, opts, defaultRecoveryOperationDeps()) +} + +func executeWithDeps(ctx context.Context, opts Options, deps recoveryOperationDeps) (report Report, err error) { + if opts.ActivePath == "" { + err := fmt.Errorf("active database path is required") + if opts.DryRun { + return Report{}, err + } + return Report{}, initialApplyFailure(phaseSourceRead, "", knownPathDescription(opts.FromPath), err) + } + if opts.FromPath == "" { + err := fmt.Errorf("--from database path is required") + if opts.DryRun { + return Report{}, err + } + return Report{}, initialApplyFailure(phaseSourceRead, knownPathDescription(opts.ActivePath), "", err) + } + + active, err := resolvePath(opts.ActivePath) + if err != nil { + if opts.DryRun { + return Report{}, err + } + return Report{}, initialApplyFailure(phaseSourceRead, knownPathDescription(opts.ActivePath), knownPathDescription(opts.FromPath), err) + } + from, err := resolvePath(opts.FromPath) + if err != nil { + if opts.DryRun { + return Report{}, err + } + return Report{}, initialApplyFailure(phaseSourceRead, active.path, knownPathDescription(opts.FromPath), err) + } + + report.ActivePath = active.path + report.BackupPath = intendedBackupDescription(active.path) + + paths := []resolvedPath{active} + if !sameFile(active, from) { + paths = append(paths, from) + } + + if opts.DryRun { + plan, diagnostics, err := planRecoveryFromImmutablePaths(ctx, paths, &report, true, deps) + if err != nil { + return report, err + } + fillReportFromPlan(&report, plan, diagnostics) + if len(diagnostics) != 0 { + return report, fmt.Errorf("recovery plan has %d diagnostic(s)", len(diagnostics)) + } + return report, nil + } + + plan, plannedActive, verifiedTempPath, diagnostics, err := planRecoveryWithActiveReservation(ctx, active, from, &report, deps) + if err != nil { + if len(diagnostics) != 0 || len(plan.InputShapes) != 0 { + fillReportFromPlan(&report, plan, diagnostics) + } + return report, ensureApplyFailureFromPath(err, phasePlanning, active.path, verifiedTempPath, from.path) + } + fillReportFromPlan(&report, plan, diagnostics) + if len(diagnostics) != 0 { + return report, enrichApplyFailuresFromPath(applyDiagnosticFailure(active.path, diagnostics), from.path) + } + backupPath, err := replaceActiveWithBackup(ctx, active.path, verifiedTempPath, plan, plannedActive, deps) + if err != nil { + err = ensureApplyFailureFromPath(err, phaseInstall, active.path, verifiedTempPath, from.path) + if backupPath != "" { + report.BackupPath = backupPath + } + if cleanupErr := deps.cleanupFileSet(verifiedTempPath); cleanupErr != nil { + var applyErr *ApplyFailure + if errors.As(err, &applyErr) { + applyErr.addCleanupError(fmt.Errorf("cleanup verified recovery destination %s after failed replacement: %w", verifiedTempPath, cleanupErr)) + } + } + return report, enrichApplyFailuresFromPath(err, from.path) + } + report.BackupPath = backupPath + return report, nil +} + +func fillReportFromPlan(report *Report, plan compat.RecoveryPlan, diagnostics []compat.Diagnostic) { + report.Shapes = append(report.Shapes, plan.InputShapes...) + report.ExactDuplicates = plan.ExactDuplicates + report.FinalCount = len(plan.Records) + report.Conflicts = append(report.Conflicts, diagnostics...) +} + +func planRecoveryFromImmutablePaths(ctx context.Context, paths []resolvedPath, report *Report, dryRun bool, deps recoveryOperationDeps) (compat.RecoveryPlan, []compat.Diagnostic, error) { + inputs := make([]compat.RecoveryInput, 0, len(paths)) + for _, path := range paths { + input, err := readImmutableRecoveryInput(ctx, path.path, dryRun, deps) + if err != nil { + return compat.RecoveryPlan{}, nil, err + } + inputs = append(inputs, input) + report.InputCounts = append(report.InputCounts, input.RowCount) + } + + plan, diagnostics, planErr := compat.PlanRecovery(inputs) + if planErr != nil { + return compat.RecoveryPlan{}, nil, planErr + } + return plan, diagnostics, nil +} + +func readImmutableRecoveryInput(ctx context.Context, path string, dryRun bool, deps recoveryOperationDeps) (compat.RecoveryInput, error) { + if err := deps.ensureSidecars(path); err != nil { + return compat.RecoveryInput{}, recoveryOpenError(path, err, dryRun) + } + planned, err := deps.fingerprintSet(path) + if err != nil { + return compat.RecoveryInput{}, recoveryOpenError(path, err, dryRun) + } + db, openErr := storage.OpenImmutableReadOnly(path) + if openErr != nil { + return compat.RecoveryInput{}, recoveryOpenError(path, openErr, dryRun) + } + input, diag, err := storage.ReadRecoveryInput(ctx, db) + closeErr := db.Close() + if err != nil { + return compat.RecoveryInput{}, errors.Join(err, closeErr) + } + if diag != nil { + return compat.RecoveryInput{}, errors.Join(diagnosticError(*diag), closeErr) + } + if closeErr != nil { + return compat.RecoveryInput{}, closeErr + } + if err := deps.ensureSidecars(path); err != nil { + return compat.RecoveryInput{}, recoveryOpenError(path, err, dryRun) + } + got, err := deps.fingerprintSet(path) + if err != nil { + return compat.RecoveryInput{}, recoveryOpenError(path, err, dryRun) + } + if !sameSQLiteFileSetSnapshot(got, planned) { + return compat.RecoveryInput{}, fmt.Errorf("immutable recovery source changed while reading %s", path) + } + return input, nil +} + +func planRecoveryWithActiveReservation(ctx context.Context, active, from resolvedPath, report *Report, deps recoveryOperationDeps) (planOut compat.RecoveryPlan, plannedOut sqliteFileSetSnapshot, tempOut string, diagnosticsOut []compat.Diagnostic, errOut error) { + if err := deps.ensureSidecars(active.path); err != nil { + return compat.RecoveryPlan{}, sqliteFileSetSnapshot{}, "", nil, preReplacementFailure(phaseSourceRead, active.path, "", fmt.Errorf("active database sidecars are unsafe for recovery apply: %w", err), nil, nil) + } + reservation, err := deps.openReservation(ctx, active.path) + if err != nil { + return compat.RecoveryPlan{}, sqliteFileSetSnapshot{}, "", nil, preReplacementFailure(phaseReservationOpen, active.path, "", err, nil, nil) + } + closed := false + defer func() { + if !closed { + if closeErr := deps.closeReservation(reservation); closeErr != nil { + closeFailure := preReplacementFailure(phaseReservationClose, active.path, tempOut, closeErr, nil, closeErr) + if errOut != nil { + errOut = errors.Join(errOut, closeFailure) + } else { + errOut = closeFailure + } + } + } + }() + + plannedSnapshot, err := deps.fingerprintSet(active.path) + if err != nil { + return compat.RecoveryPlan{}, sqliteFileSetSnapshot{}, "", nil, preReplacementFailure(phaseSourceRead, active.path, "", fmt.Errorf("fingerprint locked active database before recovery SQL read: %w", err), nil, nil) + } + plannedSnapshot.sidecars = map[string]sqliteFileSnapshot{} + activeInput, diag, err := deps.readReservedInput(ctx, reservation) + if err != nil { + return compat.RecoveryPlan{}, sqliteFileSetSnapshot{}, "", nil, preReplacementFailure(phaseSourceRead, active.path, "", fmt.Errorf("read active recovery source under reservation: %w", err), nil, nil) + } + if diag != nil { + return compat.RecoveryPlan{}, sqliteFileSetSnapshot{}, "", nil, preReplacementFailure(phaseSourceRead, active.path, "", diagnosticError(*diag), nil, nil) + } + if err := revalidateMainSnapshot(active.path, plannedSnapshot, deps); err != nil { + return compat.RecoveryPlan{}, sqliteFileSetSnapshot{}, "", nil, preReplacementFailure(phaseSourceRead, active.path, "", fmt.Errorf("active database changed during recovery SQL read: %w", err), nil, nil) + } + inputs := []compat.RecoveryInput{activeInput} + report.InputCounts = append(report.InputCounts, activeInput.RowCount) + + if !sameFile(active, from) { + strandedInput, err := readImmutableRecoveryInput(ctx, from.path, false, deps) + if err != nil { + return compat.RecoveryPlan{}, sqliteFileSetSnapshot{}, "", nil, preReplacementFailure(phaseSourceRead, active.path, "", fmt.Errorf("read stranded recovery source %s: %w", from.path, err), nil, nil) + } + inputs = append(inputs, strandedInput) + report.InputCounts = append(report.InputCounts, strandedInput.RowCount) + } + + plan, diagnostics, planErr := compat.PlanRecovery(inputs) + if planErr != nil { + return compat.RecoveryPlan{}, sqliteFileSetSnapshot{}, "", nil, preReplacementFailure(phasePlanning, active.path, "", planErr, nil, nil) + } + if len(diagnostics) != 0 { + if err := deps.closeReservation(reservation); err != nil { + closed = true + return compat.RecoveryPlan{}, sqliteFileSetSnapshot{}, "", nil, preReplacementFailure(phaseReservationClose, active.path, "", err, nil, err) + } + closed = true + return plan, sqliteFileSetSnapshot{}, "", diagnostics, applyDiagnosticFailure(active.path, diagnostics) + } + + verifiedTempPath, err := deps.createDestination(ctx, filepath.Dir(active.path), plan) + tempOut = verifiedTempPath + if err != nil { + cleanupErr := destinationConstructionCleanupErr(err) + if leakedPath := destinationConstructionTempPath(err); leakedPath != "" { + verifiedTempPath = leakedPath + tempOut = leakedPath + } + return compat.RecoveryPlan{}, sqliteFileSetSnapshot{}, "", nil, preReplacementFailure(phaseDestinationVerify, active.path, verifiedTempPath, err, cleanupErr, nil) + } + + if err := revalidateMainSnapshot(active.path, plannedSnapshot, deps); err != nil { + cleanupErr := deps.cleanupFileSet(verifiedTempPath) + return compat.RecoveryPlan{}, sqliteFileSetSnapshot{}, "", nil, preReplacementFailure(phasePreBackupValidate, active.path, verifiedTempPath, fmt.Errorf("active database changed while planning recovery apply: %w", err), cleanupErr, nil) + } + if err := deps.closeReservation(reservation); err != nil { + cleanupErr := deps.cleanupFileSet(verifiedTempPath) + closed = true + return compat.RecoveryPlan{}, sqliteFileSetSnapshot{}, "", nil, preReplacementFailure(phaseReservationClose, active.path, verifiedTempPath, err, cleanupErr, err) + } + closed = true + if err := deps.ensureSidecars(active.path); err != nil { + cleanupErr := deps.cleanupFileSet(verifiedTempPath) + return compat.RecoveryPlan{}, sqliteFileSetSnapshot{}, "", nil, preReplacementFailure(phasePreBackupValidate, active.path, verifiedTempPath, fmt.Errorf("active database sidecars changed after closing recovery reservation: %w", err), cleanupErr, nil) + } + if err := revalidateMainSnapshot(active.path, plannedSnapshot, deps); err != nil { + cleanupErr := deps.cleanupFileSet(verifiedTempPath) + return compat.RecoveryPlan{}, sqliteFileSetSnapshot{}, "", nil, preReplacementFailure(phasePreBackupValidate, active.path, verifiedTempPath, fmt.Errorf("active database changed after closing recovery reservation: %w", err), cleanupErr, nil) + } + return plan, plannedSnapshot, verifiedTempPath, diagnostics, nil +} + +type activeReservation struct { + db *sql.DB + conn *sql.Conn +} + +func openActiveReservation(ctx context.Context, activePath string) (*activeReservation, error) { + db, err := sql.Open("sqlite", "file:"+activePath+"?mode=rw&_pragma=busy_timeout(1)") + if err != nil { + return nil, fmt.Errorf("open active database for recovery apply reservation: %w", err) + } + conn, err := db.Conn(ctx) + if err != nil { + closeErr := db.Close() + return nil, fmt.Errorf("reserve active database for recovery apply: %w", errors.Join(err, closeErr)) + } + if _, err := conn.ExecContext(ctx, `BEGIN IMMEDIATE`); err != nil { + closeErr := errors.Join(conn.Close(), db.Close()) + return nil, fmt.Errorf("reserve active database writer slot for final recovery plan: %w", errors.Join(err, closeErr)) + } + return &activeReservation{db: db, conn: conn}, nil +} + +func (r *activeReservation) QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) { + return r.conn.QueryContext(ctx, query, args...) +} + +func (r *activeReservation) QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row { + return r.conn.QueryRowContext(ctx, query, args...) +} + +func (r *activeReservation) Close() error { + var errs []error + if r.conn != nil { + if _, rollbackErr := r.conn.ExecContext(context.Background(), `ROLLBACK`); rollbackErr != nil { + errs = append(errs, fmt.Errorf("rollback active recovery reservation: %w", rollbackErr)) + } + if closeErr := r.conn.Close(); closeErr != nil { + errs = append(errs, fmt.Errorf("close active recovery reservation connection: %w", closeErr)) + } + r.conn = nil + } + if r.db != nil { + if closeErr := r.db.Close(); closeErr != nil { + errs = append(errs, fmt.Errorf("close active recovery reservation database: %w", closeErr)) + } + r.db = nil + } + return errors.Join(errs...) +} + +// ApplyFailurePhase is the recovery apply stage that failed. +type ApplyFailurePhase string + +type replacementPhase = ApplyFailurePhase + +const ( + phaseDestinationSidecar replacementPhase = "destination-sidecar-cleanup" + phaseDestinationSync replacementPhase = "destination-fsync" + phaseDestinationVerify replacementPhase = "destination-verify" + phasePreBackupValidate replacementPhase = "active-pre-backup-revalidation" + phaseBackupMain replacementPhase = "backup-main" + phaseBackupSidecar replacementPhase = "backup-sidecar" + phaseBackupFileSync replacementPhase = "backup-file-fsync" + phaseBackupDirSync replacementPhase = "backup-directory-fsync" + phasePostBackupVerify replacementPhase = "backup-snapshot-verify" + phaseInstall replacementPhase = "install-active" + phaseInstallSync replacementPhase = "install-directory-fsync" + phaseRestore replacementPhase = "restore-active" + phaseSourceRead replacementPhase = "source-read" + phasePlanning replacementPhase = "planning" + phaseReservationOpen replacementPhase = "reservation-open" + phaseReservationClose replacementPhase = "reservation-close" +) + +// ApplyFailureState describes the current filesystem state after a failed apply. +type ApplyFailureState string + +type replacementState = ApplyFailureState + +const ( + replacementStateNoMutation replacementState = "no-mutation" + replacementStateNoActiveMutation replacementState = "no-active-mutation" + replacementStateBackupVisible replacementState = "backup-visible" + replacementStateBackupFileSynced replacementState = "backup-file-synced" + replacementStateBackupDirectoryDurable replacementState = "backup-directory-durable" + replacementStateBackupIntegrityVerified replacementState = "backup-integrity-verified" + replacementStateBackupDirectoryUnknown replacementState = "backup-directory-durable-integrity-unknown" + replacementStateReplacementVisible replacementState = "replacement-visible" + replacementStateReplacementDurable replacementState = "replacement-durable" + replacementStateActiveRestoredVisibleUnknown replacementState = "active-restored-visible-integrity-unknown" + replacementStateActiveRestoredDurableVerified replacementState = "active-restored-durable-verified" + replacementStateSplitUnknown replacementState = "split-or-unknown" +) + +type replacementStatus struct { + BackupPath string + NoActiveMutation bool + BackupVisible bool + BackupFileSynced bool + BackupDirectoryDurable bool + BackupIntegrityVerified bool + BackupIntegrityUnknown bool + ReplacementVisible bool + ReplacementDurable bool + RestoredActiveVisible bool + RestoredActiveDurableVerified bool + SplitUnknown bool + Milestones ApplyFailureMilestones +} + +// ApplyFailureMilestones records historical milestones reached before a failed +// apply. These are intentionally separate from the current filesystem-state +// booleans on ApplyFailure. +type ApplyFailureMilestones struct { + BackupCreated bool + BackupFileSynced bool + BackupDirectoryDurable bool + BackupIntegrityVerified bool + ReplacementVisible bool + ReplacementDurable bool + RestoredActiveVisible bool + RestoredActiveDurableVerified bool +} + +// ApplyFailure is the structured error returned by failed recovery apply +// operations after recovery planning starts. Callers should use errors.As rather +// than parse Error text. +type ApplyFailure struct { + Phase replacementPhase + State replacementState + ActivePath string + FromPath string + BackupPath string + TempPath string + CurrentRestorablePath string + Milestones ApplyFailureMilestones + NoActiveMutation bool + BackupVisible bool + BackupFileSynced bool + BackupDirectoryDurable bool + BackupIntegrityVerified bool + BackupIntegrityUnknown bool + ReplacementVisible bool + ReplacementDurable bool + RestoredActiveVisible bool + RestoredActiveDurableVerified bool + ActiveRestored bool + ActiveInstalled bool + RestorableBackup bool + RestorableBackupPath string + Cause error + RestoreErr error + CleanupErr error + CleanupErrors []error + CloseErr error +} + +type replacementError = ApplyFailure +type preReplacementError = ApplyFailure + +func (e *ApplyFailure) Error() string { + msg := fmt.Sprintf("recovery replacement failed during %s: %v; state=%s active=%s from=%s backup=%s temp=%s", e.Phase, e.Cause, e.State, e.ActivePath, e.FromPath, e.BackupPath, e.TempPath) + if e.RestorableBackup { + msg += "; restorable backup remains at " + e.RestorableBackupPath + } + if e.ActiveRestored { + msg += "; restored active" + } + if e.RestoreErr != nil { + msg += "; restore error: " + e.RestoreErr.Error() + } + if e.CleanupErr != nil { + msg += "; cleanup error: " + e.CleanupErr.Error() + } + if e.CloseErr != nil { + msg += "; close error: " + e.CloseErr.Error() + } + return msg +} + +func (e *ApplyFailure) Unwrap() []error { + errs := []error{} + if e.Cause != nil { + errs = append(errs, e.Cause) + } + if e.RestoreErr != nil { + errs = append(errs, e.RestoreErr) + } + if e.CleanupErr != nil { + errs = append(errs, e.CleanupErr) + } + if e.CloseErr != nil { + errs = append(errs, e.CloseErr) + } + return errs +} + +func (e *ApplyFailure) addCleanupError(err error) { + if err == nil { + return + } + e.CleanupErrors = append(e.CleanupErrors, err) + e.CleanupErr = errors.Join(e.CleanupErr, err) +} + +// RestorableBackupPath reports the only known manual recovery path after a +// failed apply, when the replacement sequence proved the backup durable and +// integrity-verified and left it visible. +func RestorableBackupPath(err error) (string, bool) { + var replacementErr *replacementError + if errors.As(err, &replacementErr) && replacementErr.RestorableBackupPath != "" { + return replacementErr.RestorableBackupPath, true + } + return "", false +} + +func preReplacementFailure(phase replacementPhase, activePath, tempPath string, cause, cleanupErr, closeErr error) error { + if cause == nil { + cause = errors.Join(cleanupErr, closeErr) + } + failure := &ApplyFailure{Phase: phase, State: replacementStateNoActiveMutation, ActivePath: activePath, TempPath: tempPath, NoActiveMutation: true, Cause: cause, CloseErr: closeErr} + failure.addCleanupError(cleanupErr) + return failure +} + +func initialApplyFailure(phase replacementPhase, activePath, fromPath string, cause error) error { + return &ApplyFailure{Phase: phase, State: replacementStateNoActiveMutation, ActivePath: activePath, FromPath: fromPath, NoActiveMutation: true, Cause: cause} +} + +func ensureApplyFailure(err error, phase replacementPhase, activePath, tempPath string) error { + if err == nil { + return nil + } + var failure *ApplyFailure + if errors.As(err, &failure) { + if failure.ActivePath == "" { + failure.ActivePath = activePath + } + if failure.TempPath == "" { + failure.TempPath = tempPath + } + return err + } + return preReplacementFailure(phase, activePath, tempPath, err, nil, nil) +} + +func ensureApplyFailureFromPath(err error, phase replacementPhase, activePath, tempPath, fromPath string) error { + return enrichApplyFailuresFromPath(ensureApplyFailure(err, phase, activePath, tempPath), fromPath) +} + +func enrichApplyFailuresFromPath(err error, fromPath string) error { + if err == nil || fromPath == "" { + return err + } + enrichApplyFailureTreeFromPath(err, fromPath) + return err +} + +func enrichApplyFailureTreeFromPath(err error, fromPath string) { + if err == nil { + return + } + if failure, ok := err.(*ApplyFailure); ok && failure.FromPath == "" { + failure.FromPath = fromPath + } + if wrapped, ok := err.(interface{ Unwrap() []error }); ok { + for _, child := range wrapped.Unwrap() { + enrichApplyFailureTreeFromPath(child, fromPath) + } + return + } + if wrapped, ok := err.(interface{ Unwrap() error }); ok { + enrichApplyFailureTreeFromPath(wrapped.Unwrap(), fromPath) + } +} + +func applyDiagnosticFailure(activePath string, diagnostics []compat.Diagnostic) error { + if len(diagnostics) == 0 { + return nil + } + summary := fmt.Errorf("recovery plan has %d diagnostic(s)", len(diagnostics)) + for _, diag := range diagnostics { + summary = errors.Join(summary, diagnosticError(diag)) + } + return preReplacementFailure(phasePlanning, activePath, "", summary, nil, nil) +} + +func destinationConstructionTempPath(err error) string { + var destinationErr *storage.RecoveryDestinationError + if errors.As(err, &destinationErr) { + return destinationErr.Path + } + return "" +} + +func destinationConstructionCleanupErr(err error) error { + var destinationErr *storage.RecoveryDestinationError + if errors.As(err, &destinationErr) { + return destinationErr.CleanupErr + } + return nil +} + +func replaceActiveWithBackup(ctx context.Context, activePath, verifiedTempPath string, plan compat.RecoveryPlan, plannedActive sqliteFileSetSnapshot, deps recoveryOperationDeps) (backupPath string, err error) { + defer func() { + var re *replacementError + if errors.As(err, &re) { + re.ActivePath = activePath + re.TempPath = verifiedTempPath + } + }() + activeDir := filepath.Dir(activePath) + if activeDir != filepath.Dir(verifiedTempPath) { + return "", replacementFailure(phasePreBackupValidate, replacementStatus{NoActiveMutation: true}, fmt.Errorf("recovery replacement requires same directory: active=%s temp=%s", activePath, verifiedTempPath), nil) + } + verifiedDestination, err := prepareVerifiedDestination(ctx, verifiedTempPath, plan, deps) + if err != nil { + var re *replacementError + if errors.As(err, &re) { + return re.RestorableBackupPath, re + } + return "", err + } + if err := revalidatePlannedActive(activePath, plannedActive, deps); err != nil { + return "", replacementFailure(phasePreBackupValidate, replacementStatus{NoActiveMutation: true}, fmt.Errorf("active database changed after locked recovery planning: %w", err), nil) + } + if err := deps.syncFile(activePath); err != nil { + return "", replacementFailure(phasePreBackupValidate, replacementStatus{NoActiveMutation: true}, fmt.Errorf("fsync active database before backup: %w", err), nil) + } + if err := revalidatePlannedActive(activePath, plannedActive, deps); err != nil { + return "", replacementFailure(phasePreBackupValidate, replacementStatus{NoActiveMutation: true}, fmt.Errorf("active database changed after pre-backup fsync: %w", err), nil) + } + + backupPath, movedSidecars, err := installBackup(activePath, plannedActive, deps) + if err != nil { + var re *replacementError + if errors.As(err, &re) && re.RestorableBackup { + return re.RestorableBackupPath, err + } + return "", err + } + backupStatus := backupVisibleStatus(backupPath) + if err := ensureNoActiveSQLiteSidecars(activePath); err != nil { + outcome := restoreBackup(activePath, backupPath, movedSidecars, plannedActive, deps) + return replacementBackupPath(outcome, backupPath), replacementFailure(phasePostBackupVerify, withRestoreOutcome(backupStatus, outcome), err, outcome.err) + } + if err := verifyBackupMatchesPlannedSnapshot(backupPath, plannedActive, deps); err != nil { + outcome := restoreBackup(activePath, backupPath, movedSidecars, plannedActive, deps) + return replacementBackupPath(outcome, backupPath), replacementFailure(phasePostBackupVerify, withRestoreOutcome(backupStatus, outcome), err, outcome.err) + } + if err := deps.syncFile(backupPath); err != nil { + outcome := restoreBackup(activePath, backupPath, movedSidecars, plannedActive, deps) + return replacementBackupPath(outcome, backupPath), replacementFailure(phaseBackupFileSync, withRestoreOutcome(backupStatus, outcome), fmt.Errorf("fsync recovery backup file: %w", err), outcome.err) + } + backupStatus = backupFileSyncedStatus(backupPath) + if err := deps.syncDir(activeDir); err != nil { + outcome := restoreBackup(activePath, backupPath, movedSidecars, plannedActive, deps) + return replacementBackupPath(outcome, backupPath), replacementFailure(phaseBackupDirSync, withRestoreOutcome(backupStatus, outcome), err, outcome.err) + } + backupStatus = backupDurableStatus(backupPath) + if err := verifyBackupMatchesPlannedSnapshot(backupPath, plannedActive, deps); err != nil { + unknownBackup := backupUnknownStatus(backupPath) + outcome := restoreBackup(activePath, backupPath, movedSidecars, plannedActive, deps) + return replacementBackupPath(outcome, backupPath), replacementFailure(phasePostBackupVerify, withRestoreOutcome(unknownBackup, outcome), err, outcome.err) + } + backupStatus = backupVerifiedStatus(backupPath) + + if err := deps.ensureSidecars(activePath); err != nil { + outcome := restoreBackup(activePath, backupPath, movedSidecars, plannedActive, deps) + return replacementBackupPath(outcome, backupPath), replacementFailure(phaseInstall, withRestoreOutcome(backupStatus, outcome), err, outcome.err) + } + if _, err := os.Lstat(activePath); err == nil || !os.IsNotExist(err) { + if err == nil { + err = fmt.Errorf("active path unexpectedly exists before installing replacement: %s", activePath) + } + outcome := restoreBackup(activePath, backupPath, movedSidecars, plannedActive, deps) + return replacementBackupPath(outcome, backupPath), replacementFailure(phaseInstall, withRestoreOutcome(backupStatus, outcome), err, outcome.err) + } + if err := deps.installMove(verifiedTempPath, activePath); err != nil { + outcome := restoreBackup(activePath, backupPath, movedSidecars, plannedActive, deps) + return replacementBackupPath(outcome, backupPath), replacementFailure(phaseInstall, withRestoreOutcome(backupStatus, outcome), err, outcome.err) + } + if err := deps.syncDir(activeDir); err != nil { + // The replacement rename succeeded, but the final directory fsync did not. + // Keep the last known durable backup instead of consuming it for a restore + // that could make the crash-recovery state less truthful. + installed := backupStatus + installed.ReplacementVisible = true + installed.Milestones.ReplacementVisible = true + return backupPath, replacementFailure(phaseInstallSync, installed, err, nil) + } + installed := backupStatus + installed.ReplacementVisible = true + installed.Milestones.ReplacementVisible = true + if err := revalidateSnapshot(activePath, verifiedDestination, deps); err != nil { + return backupPath, replacementFailure(phaseInstallSync, installed, fmt.Errorf("installed active database differs from verified recovery destination: %w", err), nil) + } + installed.ReplacementDurable = true + installed.Milestones.ReplacementDurable = true + if err := verifyBackupMatchesPlannedSnapshot(backupPath, plannedActive, deps); err != nil { + unknownBackup := installed + unknownBackup.BackupIntegrityVerified = false + unknownBackup.BackupIntegrityUnknown = true + return backupPath, replacementFailure(phaseInstallSync, unknownBackup, fmt.Errorf("durable backup changed after installing replacement: %w", err), nil) + } + return backupPath, nil +} + +func prepareVerifiedDestination(ctx context.Context, verifiedTempPath string, plan compat.RecoveryPlan, deps recoveryOperationDeps) (sqliteFileSetSnapshot, error) { + status := replacementStatus{NoActiveMutation: true} + if err := deps.ensureSidecars(verifiedTempPath); err != nil { + return sqliteFileSetSnapshot{}, replacementFailure(phaseDestinationSidecar, status, err, nil) + } + if err := deps.verifyDestination(ctx, verifiedTempPath, plan); err != nil { + return sqliteFileSetSnapshot{}, replacementFailure(phaseDestinationVerify, status, err, nil) + } + if err := deps.syncFile(verifiedTempPath); err != nil { + return sqliteFileSetSnapshot{}, replacementFailure(phaseDestinationSync, status, err, nil) + } + if err := deps.ensureSidecars(verifiedTempPath); err != nil { + return sqliteFileSetSnapshot{}, replacementFailure(phaseDestinationSidecar, status, err, nil) + } + fingerprint, err := deps.fingerprintSet(verifiedTempPath) + if err != nil { + return sqliteFileSetSnapshot{}, replacementFailure(phaseDestinationVerify, status, fmt.Errorf("fingerprint verified recovery destination: %w", err), nil) + } + return fingerprint, nil +} + +func moveFailureStatus(candidate string, err error) replacementStatus { + status := replacementStatus{BackupPath: candidate, NoActiveMutation: true} + var unsupported noClobberUnsupportedError + if errors.As(err, &unsupported) { + status.NoActiveMutation = true + } + return status +} + +func installBackup(activePath string, plannedActive sqliteFileSetSnapshot, deps recoveryOperationDeps) (backupPath string, movedSidecars []string, err error) { + if err := deps.ensureSidecars(activePath); err != nil { + return "", nil, replacementFailure(phaseBackupMain, replacementStatus{NoActiveMutation: true}, err, nil) + } + for i := 0; i < 100; i++ { + candidate, candidateErr := backupPathCandidate(activePath, deps) + if candidateErr != nil { + return "", nil, replacementFailure(phaseBackupMain, replacementStatus{NoActiveMutation: true}, candidateErr, nil) + } + moveErr := deps.noClobberMove(activePath, candidate) + if moveErr == nil { + backupPath = candidate + break + } + if os.IsExist(moveErr) { + continue + } + status := moveFailureStatus(candidate, moveErr) + return "", nil, replacementFailure(phaseBackupMain, status, fmt.Errorf("move active database to backup candidate %s: %w", candidate, moveErr), nil) + } + if backupPath == "" { + return "", nil, replacementFailure(phaseBackupMain, replacementStatus{NoActiveMutation: true}, fmt.Errorf("could not choose a unique recovery backup path for %s", activePath), nil) + } + + for _, suffix := range sortedSnapshotSidecars(plannedActive) { + if err := deps.noClobberMove(activePath+suffix, backupPath+suffix); err != nil { + outcome := restoreBackup(activePath, backupPath, movedSidecars, plannedActive, deps) + return backupPath, movedSidecars, replacementFailure(phaseBackupSidecar, withRestoreOutcome(backupVisibleStatus(backupPath), outcome), fmt.Errorf("move sidecar %s: %w", suffix, err), outcome.err) + } + movedSidecars = append(movedSidecars, suffix) + } + return backupPath, movedSidecars, nil +} + +func replacementFailure(phase replacementPhase, status replacementStatus, cause, restoreErr error) *replacementError { + state := replacementStateNoMutation + switch { + case status.RestoredActiveDurableVerified: + state = replacementStateActiveRestoredDurableVerified + case status.RestoredActiveVisible: + state = replacementStateActiveRestoredVisibleUnknown + case status.ReplacementDurable: + state = replacementStateReplacementDurable + case status.ReplacementVisible: + state = replacementStateReplacementVisible + case status.SplitUnknown || restoreErr != nil: + state = replacementStateSplitUnknown + case status.BackupIntegrityUnknown: + state = replacementStateBackupDirectoryUnknown + case status.BackupIntegrityVerified: + state = replacementStateBackupIntegrityVerified + case status.BackupDirectoryDurable: + state = replacementStateBackupDirectoryDurable + case status.BackupFileSynced: + state = replacementStateBackupFileSynced + case status.BackupVisible: + state = replacementStateBackupVisible + case status.NoActiveMutation: + state = replacementStateNoActiveMutation + } + restorableBackup := status.BackupPath != "" && status.BackupVisible && status.BackupDirectoryDurable && status.BackupIntegrityVerified && !status.BackupIntegrityUnknown && !status.RestoredActiveVisible && !status.RestoredActiveDurableVerified + restorablePath := mapRestorableBackupPath(restorableBackup, status.BackupPath) + return &replacementError{ + Phase: phase, + State: state, + BackupPath: status.BackupPath, + CurrentRestorablePath: restorablePath, + Milestones: status.Milestones, + NoActiveMutation: status.NoActiveMutation, + BackupVisible: status.BackupVisible, + BackupFileSynced: status.BackupFileSynced, + BackupDirectoryDurable: status.BackupDirectoryDurable, + BackupIntegrityVerified: status.BackupIntegrityVerified && !status.BackupIntegrityUnknown, + BackupIntegrityUnknown: status.BackupIntegrityUnknown, + ReplacementVisible: status.ReplacementVisible || status.ReplacementDurable, + ReplacementDurable: status.ReplacementDurable, + RestoredActiveVisible: status.RestoredActiveVisible || status.RestoredActiveDurableVerified, + RestoredActiveDurableVerified: status.RestoredActiveDurableVerified, + ActiveRestored: status.RestoredActiveDurableVerified, + ActiveInstalled: status.ReplacementVisible || status.ReplacementDurable, + RestorableBackup: restorableBackup, + RestorableBackupPath: restorablePath, + Cause: cause, + RestoreErr: restoreErr, + } +} + +func mapRestorableBackupPath(restorable bool, backupPath string) string { + if restorable { + return backupPath + } + return "" +} + +type restoreOutcome struct { + activeVisible bool + activeDurableVerified bool + err error +} + +func replacementBackupPath(outcome restoreOutcome, backupPath string) string { + if outcome.activeVisible { + return "" + } + return backupPath +} + +func backupVisibleStatus(backupPath string) replacementStatus { + return replacementStatus{BackupPath: backupPath, BackupVisible: backupPath != "", Milestones: ApplyFailureMilestones{BackupCreated: backupPath != ""}} +} + +func backupFileSyncedStatus(backupPath string) replacementStatus { + return replacementStatus{BackupPath: backupPath, BackupVisible: true, BackupFileSynced: true, Milestones: ApplyFailureMilestones{BackupCreated: true, BackupFileSynced: true}} +} + +func backupDurableStatus(backupPath string) replacementStatus { + return replacementStatus{BackupPath: backupPath, BackupVisible: true, BackupFileSynced: true, BackupDirectoryDurable: true, Milestones: ApplyFailureMilestones{BackupCreated: true, BackupFileSynced: true, BackupDirectoryDurable: true}} +} + +func backupVerifiedStatus(backupPath string) replacementStatus { + return replacementStatus{BackupPath: backupPath, BackupVisible: true, BackupFileSynced: true, BackupDirectoryDurable: true, BackupIntegrityVerified: true, Milestones: ApplyFailureMilestones{BackupCreated: true, BackupFileSynced: true, BackupDirectoryDurable: true, BackupIntegrityVerified: true}} +} + +func backupUnknownStatus(backupPath string) replacementStatus { + return replacementStatus{BackupPath: backupPath, BackupVisible: true, BackupFileSynced: true, BackupDirectoryDurable: true, BackupIntegrityUnknown: true, Milestones: ApplyFailureMilestones{BackupCreated: true, BackupFileSynced: true, BackupDirectoryDurable: true}} +} + +func withRestoreOutcome(status replacementStatus, outcome restoreOutcome) replacementStatus { + if outcome.activeVisible { + status.BackupVisible = false + status.BackupFileSynced = false + status.BackupDirectoryDurable = false + status.BackupIntegrityVerified = false + status.BackupIntegrityUnknown = false + status.RestoredActiveVisible = true + status.Milestones.RestoredActiveVisible = true + } + if outcome.activeDurableVerified { + status.RestoredActiveDurableVerified = true + status.Milestones.RestoredActiveDurableVerified = true + return status + } + if outcome.err != nil && !outcome.activeVisible { + status.SplitUnknown = true + } + return status +} + +func restoreBackup(activePath, backupPath string, backupSidecars []string, plannedActive sqliteFileSetSnapshot, deps recoveryOperationDeps) restoreOutcome { + if backupPath == "" { + return restoreOutcome{err: fmt.Errorf("no backup path available to restore")} + } + if err := deps.ensureSidecars(activePath); err != nil { + return restoreOutcome{err: fmt.Errorf("active namespace has unexpected sidecars before restore: %w", err)} + } + for i := len(backupSidecars) - 1; i >= 0; i-- { + suffix := backupSidecars[i] + if err := deps.restoreMove(backupPath+suffix, activePath+suffix); err != nil { + return restoreOutcome{err: fmt.Errorf("restore active sidecar %s from backup: %w", suffix, err)} + } + } + if err := deps.restoreMove(backupPath, activePath); err != nil { + return restoreOutcome{err: fmt.Errorf("restore active database from backup: %w", err)} + } + if err := deps.syncFile(activePath); err != nil { + return restoreOutcome{activeVisible: true, err: fmt.Errorf("fsync restored active file: %w", err)} + } + if err := deps.syncDir(filepath.Dir(activePath)); err != nil { + return restoreOutcome{activeVisible: true, err: fmt.Errorf("fsync restored active directory: %w", err)} + } + if err := revalidatePlannedActive(activePath, plannedActive, deps); err != nil { + return restoreOutcome{activeVisible: true, err: fmt.Errorf("verify restored active database: %w", err)} + } + return restoreOutcome{activeVisible: true, activeDurableVerified: true} +} + +func ensureSourceSidecarsSafe(dbPath string) error { + for _, suffix := range sqliteSidecarSuffixes() { + path := dbPath + suffix + if _, err := os.Lstat(path); err == nil { + if suffix == "-wal" { + return fmt.Errorf("%w: unexpected SQLite sidecar namespace entry %s", storage.ErrImmutableReadOnlyWALUnsafe, path) + } + return fmt.Errorf("unexpected SQLite sidecar namespace entry %s", path) + } else if !os.IsNotExist(err) { + return fmt.Errorf("lstat SQLite sidecar %s: %w", path, err) + } + } + return nil +} + +func ensureNoActiveSQLiteSidecars(dbPath string) error { + if err := ensureSourceSidecarsSafe(dbPath); err != nil { + return fmt.Errorf("unexpected active sidecar appeared after backup: %w", err) + } + return nil +} + +func verifyBackupMatchesPlannedSnapshot(backupPath string, planned sqliteFileSetSnapshot, deps recoveryOperationDeps) error { + got, err := deps.fingerprintSet(backupPath) + if err != nil { + return fmt.Errorf("snapshot recovery backup %s: %w", backupPath, err) + } + if !sameSQLiteFileSetSnapshot(got, planned) { + return fmt.Errorf("recovery backup %s does not match locked active snapshot", backupPath) + } + return nil +} + +func revalidatePlannedActive(activePath string, planned sqliteFileSetSnapshot, deps recoveryOperationDeps) error { + if err := deps.ensureSidecars(activePath); err != nil { + return err + } + return revalidateSnapshot(activePath, planned, deps) +} + +func revalidateSnapshot(path string, planned sqliteFileSetSnapshot, deps recoveryOperationDeps) error { + got, err := deps.fingerprintSet(path) + if err != nil { + return err + } + if !sameSQLiteFileSetSnapshot(got, planned) { + return fmt.Errorf("file set no longer matches planned snapshot") + } + return nil +} + +func revalidateMainSnapshot(path string, planned sqliteFileSetSnapshot, deps recoveryOperationDeps) error { + got, err := deps.fingerprintSet(path) + if err != nil { + return err + } + if !sameSQLiteFileSnapshot(got.main, planned.main) { + return fmt.Errorf("main file no longer matches planned snapshot") + } + return nil +} + +type sqliteFileSetSnapshot struct { + main sqliteFileSnapshot + sidecars map[string]sqliteFileSnapshot +} + +type sqliteFileSnapshot struct { + sha256 string + size int64 + mode os.FileMode + modTime time.Time + info os.FileInfo +} + +func snapshotSQLiteFileSet(dbPath string) (sqliteFileSetSnapshot, error) { + main, err := snapshotSQLiteFile(dbPath) + if err != nil { + return sqliteFileSetSnapshot{}, err + } + sidecars := map[string]sqliteFileSnapshot{} + for _, suffix := range sqliteSidecarSuffixes() { + path := dbPath + suffix + if _, err := os.Lstat(path); os.IsNotExist(err) { + continue + } else if err != nil { + return sqliteFileSetSnapshot{}, fmt.Errorf("lstat sidecar %s: %w", path, err) + } + snap, err := snapshotSQLiteFile(path) + if err != nil { + return sqliteFileSetSnapshot{}, fmt.Errorf("snapshot sidecar %s: %w", path, err) + } + sidecars[suffix] = snap + } + return sqliteFileSetSnapshot{main: main, sidecars: sidecars}, nil +} + +type snapshotFile interface { + io.Reader + Stat() (os.FileInfo, error) + Close() error +} + +func snapshotSQLiteFile(path string) (sqliteFileSnapshot, error) { + pathInfo, err := os.Lstat(path) + if err != nil { + return sqliteFileSnapshot{}, err + } + if !pathInfo.Mode().IsRegular() { + return sqliteFileSnapshot{}, fmt.Errorf("%s is not a regular file", path) + } + file, err := os.Open(path) + if err != nil { + return sqliteFileSnapshot{}, err + } + return snapshotSQLiteOpenFile(path, pathInfo, file) +} + +func snapshotSQLiteOpenFile(path string, pathInfo os.FileInfo, file snapshotFile) (snapshot sqliteFileSnapshot, err error) { + defer func() { + if closeErr := file.Close(); closeErr != nil { + err = errors.Join(err, fmt.Errorf("close fingerprint file %s: %w", path, closeErr)) + } + }() + openInfo, err := file.Stat() + if err != nil { + return sqliteFileSnapshot{}, err + } + if !os.SameFile(pathInfo, openInfo) { + return sqliteFileSnapshot{}, fmt.Errorf("%s changed between lstat and open", path) + } + h := sha256.New() + if _, err := io.Copy(h, file); err != nil { + return sqliteFileSnapshot{}, err + } + closeReadInfo, err := file.Stat() + if err != nil { + return sqliteFileSnapshot{}, err + } + if !os.SameFile(openInfo, closeReadInfo) || closeReadInfo.Size() != openInfo.Size() || !closeReadInfo.ModTime().Equal(openInfo.ModTime()) { + return sqliteFileSnapshot{}, fmt.Errorf("%s changed while fingerprinting", path) + } + pathInfoAfter, err := os.Lstat(path) + if err != nil { + return sqliteFileSnapshot{}, err + } + if !os.SameFile(openInfo, pathInfoAfter) { + return sqliteFileSnapshot{}, fmt.Errorf("%s changed after fingerprinting", path) + } + return sqliteFileSnapshot{ + sha256: hex.EncodeToString(h.Sum(nil)), + size: openInfo.Size(), + mode: openInfo.Mode(), + modTime: openInfo.ModTime(), + info: openInfo, + }, nil +} + +func sameSQLiteFileSetSnapshot(left, right sqliteFileSetSnapshot) bool { + if !sameSQLiteFileSnapshot(left.main, right.main) { + return false + } + if len(left.sidecars) != len(right.sidecars) { + return false + } + for suffix, leftSnap := range left.sidecars { + rightSnap, ok := right.sidecars[suffix] + if !ok || !sameSQLiteFileSnapshot(leftSnap, rightSnap) { + return false + } + } + return true +} + +func sameSQLiteFileSnapshot(left, right sqliteFileSnapshot) bool { + if left.info == nil || right.info == nil { + return false + } + return left.sha256 == right.sha256 && left.size == right.size && os.SameFile(left.info, right.info) +} + +func sortedSnapshotSidecars(snapshot sqliteFileSetSnapshot) []string { + keys := make([]string, 0, len(snapshot.sidecars)) + for key := range snapshot.sidecars { + keys = append(keys, key) + } + if len(keys) == 2 && keys[0] > keys[1] { + keys[0], keys[1] = keys[1], keys[0] + } + return keys +} + +func backupPathCandidate(activePath string, deps recoveryOperationDeps) (string, error) { + suffix, err := deps.randomHex(3) + if err != nil { + return "", err + } + base := filepath.Base(activePath) + if !strings.HasPrefix(base, ".") { + base = "." + base + } + return filepath.Join(filepath.Dir(activePath), fmt.Sprintf("%s.backup-%s-%s", base, time.Now().UTC().Format("20060102T150405Z"), suffix)), nil +} + +func intendedBackupDescription(activePath string) string { + base := filepath.Base(activePath) + if !strings.HasPrefix(base, ".") { + base = "." + base + } + return filepath.Join(filepath.Dir(activePath), base+".backup--") +} + +func randomHex(n int) (string, error) { + buf := make([]byte, n) + if _, err := rand.Read(buf); err != nil { + return "", fmt.Errorf("read random recovery backup suffix: %w", err) + } + return hex.EncodeToString(buf), nil +} + +type noClobberUnsupportedError struct { + oldPath string + newPath string + cause error +} + +func (e noClobberUnsupportedError) Error() string { + if e.cause != nil { + return fmt.Sprintf("atomic no-clobber move unsupported before mutation: old=%s new=%s: %v", e.oldPath, e.newPath, e.cause) + } + return fmt.Sprintf("atomic no-clobber move unsupported before mutation: old=%s new=%s", e.oldPath, e.newPath) +} + +func (e noClobberUnsupportedError) Unwrap() error { return e.cause } + +func removeSQLiteFileSet(dbPath string) error { + var errs []error + for _, path := range append([]string{dbPath}, sqliteSidecarPaths(dbPath)...) { + if removeErr := os.Remove(path); removeErr != nil && !os.IsNotExist(removeErr) { + errs = append(errs, fmt.Errorf("remove SQLite recovery file %s: %w", path, removeErr)) + } + } + return errors.Join(errs...) +} + +func sqliteSidecarPaths(dbPath string) []string { + suffixes := sqliteSidecarSuffixes() + paths := make([]string, 0, len(suffixes)) + for _, suffix := range suffixes { + paths = append(paths, dbPath+suffix) + } + return paths +} + +func sqliteSidecarSuffixes() []string { + return []string{"-wal", "-shm", "-journal"} +} + +func syncFile(path string) error { + f, err := os.Open(path) + if err != nil { + return err + } + syncErr := f.Sync() + closeErr := f.Close() + return errors.Join(syncErr, closeErr) +} + +func syncDir(path string) error { + f, err := os.Open(path) + if err != nil { + return err + } + syncErr := f.Sync() + closeErr := f.Close() + return errors.Join(syncErr, closeErr) +} + +func resolvePath(path string) (resolvedPath, error) { + abs, err := filepath.Abs(path) + if err != nil { + return resolvedPath{}, fmt.Errorf("resolve database path %s: %w", path, err) + } + info, err := os.Lstat(abs) + if err != nil { + if os.IsNotExist(err) { + return resolvedPath{path: abs}, nil + } + return resolvedPath{}, fmt.Errorf("stat database path %s: %w", abs, err) + } + resolved, err := filepath.EvalSymlinks(abs) + if err != nil { + if strings.Contains(err.Error(), "too many links") { + resolved = abs + } else { + return resolvedPath{}, fmt.Errorf("resolve database symlink %s: %w", abs, err) + } + } + info, err = os.Stat(resolved) + if err != nil { + if os.IsNotExist(err) { + return resolvedPath{path: resolved}, nil + } + return resolvedPath{}, fmt.Errorf("stat database path %s: %w", resolved, err) + } + return resolvedPath{path: resolved, info: info}, nil +} + +func knownPathDescription(path string) string { + if path == "" { + return "" + } + abs, err := filepath.Abs(path) + if err != nil { + return path + } + return abs +} + +func sameFile(left, right resolvedPath) bool { + if left.info == nil || right.info == nil { + return false + } + return os.SameFile(left.info, right.info) +} + +func recoveryOpenError(path string, err error, dryRun bool) error { + mode := "recovery apply" + if dryRun { + mode = "recovery dry-run" + } + if errors.Is(err, storage.ErrImmutableReadOnlyWALUnsafe) { + return fmt.Errorf("%s cannot inspect %s without side effects while its WAL has uncheckpointed frames; close the writer or checkpoint the database, then retry: %w", mode, path, err) + } + return fmt.Errorf("open %s for %s without side effects: %w", path, mode, err) +} + +func diagnosticError(diag compat.Diagnostic) error { + return fmt.Errorf("%s: %s", diag.Code, diag.Summary) +} diff --git a/internal/recovery/recovery_test.go b/internal/recovery/recovery_test.go new file mode 100644 index 0000000..faae251 --- /dev/null +++ b/internal/recovery/recovery_test.go @@ -0,0 +1,2076 @@ +package recovery + +import ( + "bytes" + "context" + "crypto/sha256" + "errors" + "fmt" + "os" + "path/filepath" + "reflect" + "sort" + "strings" + "testing" + "time" + + "github.com/pablontiv/backscroll/internal/compat" + "github.com/pablontiv/backscroll/internal/storage" +) + +func TestRecoverAtomicallyReplacesAndPreservesActiveBackup(t *testing.T) { + dir := t.TempDir() + activePath := filepath.Join(dir, "active.db") + fromPath := filepath.Join(dir, "stranded.db") + createRecoveryDB(t, activePath, []storage.IndexedMessage{{ + Ordinal: 0, + Role: "user", + Text: "original active row", + UUID: "11111111-1111-4111-8111-111111111111", + Timestamp: "2026-08-18T00:00:00Z", + ContentType: "text", + }}) + createRecoveryDB(t, fromPath, []storage.IndexedMessage{{ + Ordinal: 0, + Role: "assistant", + Text: "rescued stranded row", + UUID: "22222222-2222-4222-8222-222222222222", + Timestamp: "2026-08-18T00:01:00Z", + ContentType: "text", + }}) + + activeBefore := snapshotRecoveryDBFile(t, activePath) + strandedBefore := snapshotRecoveryDBFile(t, fromPath) + + dryRun, err := Execute(context.Background(), Options{ActivePath: activePath, FromPath: fromPath, DryRun: true}) + if err != nil { + t.Fatalf("Execute dry run: %v", err) + } + if dryRun.FinalCount != 2 || !reflect.DeepEqual(dryRun.InputCounts, []int{1, 1}) { + t.Fatalf("dry-run report = %+v, want final count 2 and one row per input", dryRun) + } + assertRecoveryDBFileSnapshot(t, "active after dry-run", activePath, activeBefore) + assertRecoveryDBFileSnapshot(t, "stranded after dry-run", fromPath, strandedBefore) + + report, err := Execute(context.Background(), Options{ActivePath: activePath, FromPath: fromPath}) + if err != nil { + t.Fatalf("Execute apply: %v", err) + } + if report.ActivePath != canonicalRecoveryTestPath(t, activePath) { + t.Fatalf("ActivePath = %q, want resolved active path", report.ActivePath) + } + if report.BackupPath == "" || report.BackupPath == activePath || filepath.Dir(report.BackupPath) != filepath.Dir(report.ActivePath) { + t.Fatalf("BackupPath = %q, want unique sibling of active %q", report.BackupPath, report.ActivePath) + } + if !strings.Contains(filepath.Base(report.BackupPath), filepath.Base(report.ActivePath)+".backup-") { + t.Fatalf("BackupPath = %q, want preserved active basename with backup suffix", report.BackupPath) + } + if report.FinalCount != 2 || !reflect.DeepEqual(report.InputCounts, []int{1, 1}) { + t.Fatalf("apply report = %+v, want final count 2 and one row per input", report) + } + backup := snapshotRecoveryDBFile(t, report.BackupPath) + if !bytes.Equal(backup.main.data, activeBefore.main.data) { + t.Fatalf("backup bytes differ from original active bytes") + } + assertRecoveryDBFileSnapshot(t, "stranded after apply", fromPath, strandedBefore) + assertRecoveryTexts(t, activePath, []string{"original active row", "rescued stranded row"}) + if _, err := os.Stat(report.BackupPath); err != nil { + t.Fatalf("backup disappeared after successful recovery: %v", err) + } +} + +func TestRecoverStrandedSourceIsReadOnly(t *testing.T) { + t.Run("conflict failure", func(t *testing.T) { + dir := t.TempDir() + activePath := filepath.Join(dir, "active.db") + fromPath := filepath.Join(dir, "stranded.db") + createRecoveryDB(t, activePath, []storage.IndexedMessage{{ + Ordinal: 0, + Role: "user", + Text: "active conflicting payload", + UUID: "33333333-3333-4333-8333-333333333333", + Timestamp: "2026-08-18T00:00:00Z", + ContentType: "text", + }}) + createRecoveryDB(t, fromPath, []storage.IndexedMessage{{ + Ordinal: 0, + Role: "user", + Text: "stranded conflicting payload", + UUID: "33333333-3333-4333-8333-333333333333", + Timestamp: "2026-08-18T00:00:00Z", + ContentType: "text", + }}) + activeBefore := snapshotRecoveryDBFile(t, activePath) + strandedBefore := snapshotRecoveryDBFile(t, fromPath) + + _, err := Execute(context.Background(), Options{ActivePath: activePath, FromPath: fromPath}) + if err == nil { + t.Fatal("Execute conflicting apply succeeded; want complete-plan failure before replacement") + } + assertRecoveryDBFileSnapshot(t, "active after conflict", activePath, activeBefore) + assertRecoveryDBFileSnapshot(t, "stranded after conflict", fromPath, strandedBefore) + }) + + t.Run("replacement failure", func(t *testing.T) { + dir := t.TempDir() + activePath := filepath.Join(dir, "active.db") + fromPath := filepath.Join(dir, "stranded.db") + createRecoveryDB(t, activePath, []storage.IndexedMessage{{ + Ordinal: 0, + Role: "user", + Text: "active survives replacement failure", + UUID: "44444444-4444-4444-8444-444444444444", + Timestamp: "2026-08-18T00:00:00Z", + ContentType: "text", + }}) + createRecoveryDB(t, fromPath, []storage.IndexedMessage{{ + Ordinal: 0, + Role: "assistant", + Text: "stranded remains immutable", + UUID: "55555555-5555-4555-8555-555555555555", + Timestamp: "2026-08-18T00:01:00Z", + ContentType: "text", + }}) + activeBefore := snapshotRecoveryDBFile(t, activePath) + strandedBefore := snapshotRecoveryDBFile(t, fromPath) + + resolvedActivePath := canonicalRecoveryTestPath(t, activePath) + deps := defaultRecoveryOperationDeps() + originalMove := deps.installMove + deps.installMove = func(oldPath, newPath string) error { + if strings.Contains(filepath.Base(oldPath), ".backscroll-recover-") && newPath == resolvedActivePath { + return fmt.Errorf("injected replacement rename failure") + } + return originalMove(oldPath, newPath) + } + + _, err := executeWithDeps(context.Background(), Options{ActivePath: activePath, FromPath: fromPath}, deps) + if err == nil { + t.Fatal("Execute with replacement rename failure succeeded; want explicit failure") + } + var failure *ApplyFailure + if !errors.As(err, &failure) || !failure.ActiveRestored { + t.Fatalf("Execute error = %T %[1]v, want ApplyFailure with ActiveRestored", err) + } + assertRecoveryDBFileSnapshot(t, "active after replacement failure", activePath, activeBefore) + assertRecoveryDBFileSnapshot(t, "stranded after replacement failure", fromPath, strandedBefore) + }) +} + +func TestRecoverReplacementFailureRestoresActive(t *testing.T) { + dir := t.TempDir() + activePath := filepath.Join(dir, "active.db") + fromPath := filepath.Join(dir, "stranded.db") + createRecoveryDB(t, activePath, []storage.IndexedMessage{{ + Ordinal: 0, + Role: "user", + Text: "active before final rename failure", + UUID: "66666666-6666-4666-8666-666666666666", + Timestamp: "2026-08-18T00:00:00Z", + ContentType: "text", + }}) + createRecoveryDB(t, fromPath, []storage.IndexedMessage{{ + Ordinal: 0, + Role: "assistant", + Text: "not installed because final rename fails", + UUID: "77777777-7777-4777-8777-777777777777", + Timestamp: "2026-08-18T00:01:00Z", + ContentType: "text", + }}) + activeBefore := snapshotRecoveryDBFile(t, activePath) + + resolvedActivePath := canonicalRecoveryTestPath(t, activePath) + deps := defaultRecoveryOperationDeps() + originalMove := deps.installMove + deps.installMove = func(oldPath, newPath string) error { + if strings.Contains(filepath.Base(oldPath), ".backscroll-recover-") && newPath == resolvedActivePath { + return fmt.Errorf("injected final rename failure") + } + return originalMove(oldPath, newPath) + } + + _, err := executeWithDeps(context.Background(), Options{ActivePath: activePath, FromPath: fromPath}, deps) + if err == nil { + t.Fatal("Execute with final rename failure succeeded; want failure") + } + var failure *ApplyFailure + if !errors.As(err, &failure) { + t.Fatalf("error %T %[1]v, want *ApplyFailure", err) + } + if failure.FromPath != canonicalRecoveryTestPath(t, fromPath) { + t.Fatalf("failure FromPath = %q, want canonical stranded path", failure.FromPath) + } + if !failure.ActiveRestored { + t.Fatalf("ActiveRestored = false; want restored active after final rename failure") + } + assertRecoveryDBFileSnapshot(t, "active after final rename failure", activePath, activeBefore) +} + +func TestRecoverNeverDeletesBackup(t *testing.T) { + dir := t.TempDir() + activePath := filepath.Join(dir, "active.db") + fromPath := filepath.Join(dir, "stranded.db") + createRecoveryDB(t, activePath, []storage.IndexedMessage{{ + Ordinal: 0, + Role: "user", + Text: "active backed up once", + UUID: "88888888-8888-4888-8888-888888888888", + Timestamp: "2026-08-18T00:00:00Z", + ContentType: "text", + }}) + createRecoveryDB(t, fromPath, []storage.IndexedMessage{{ + Ordinal: 0, + Role: "assistant", + Text: "stranded installed once", + UUID: "99999999-9999-4999-8999-999999999999", + Timestamp: "2026-08-18T00:01:00Z", + ContentType: "text", + }}) + + first, err := Execute(context.Background(), Options{ActivePath: activePath, FromPath: fromPath}) + if err != nil { + t.Fatalf("first recovery: %v", err) + } + second, err := Execute(context.Background(), Options{ActivePath: activePath, FromPath: activePath}) + if err != nil { + t.Fatalf("same-path recovery after first backup: %v", err) + } + if first.BackupPath == second.BackupPath { + t.Fatalf("backup paths reused: %q", first.BackupPath) + } + for _, path := range []string{first.BackupPath, second.BackupPath} { + if _, err := os.Stat(path); err != nil { + t.Fatalf("backup %s missing after later recovery: %v", path, err) + } + } +} + +func TestRecoverReplacementFaultsAreOperationLocalAndTyped(t *testing.T) { + for _, tt := range []struct { + name string + configure func(t *testing.T, activePath string, deps *recoveryOperationDeps) + wantPhase replacementPhase + }{ + { + name: "first backup operation fails before consuming active", + configure: func(t *testing.T, activePath string, deps *recoveryOperationDeps) { + resolvedActive := canonicalRecoveryTestPath(t, activePath) + deps.noClobberMove = func(oldPath, newPath string) error { + if oldPath == resolvedActive { + return fmt.Errorf("injected backup main failure") + } + return noClobberMove(oldPath, newPath) + } + }, + wantPhase: phaseBackupMain, + }, + { + name: "destination fsync failure happens before install", + configure: func(t *testing.T, activePath string, deps *recoveryOperationDeps) { + deps.syncFile = func(path string) error { return fmt.Errorf("injected destination fsync failure") } + }, + wantPhase: phaseDestinationSync, + }, + { + name: "first backup directory fsync restores active", + configure: func(t *testing.T, activePath string, deps *recoveryOperationDeps) { + deps.syncDir = func(path string) error { return fmt.Errorf("injected first directory fsync failure") } + }, + wantPhase: phaseBackupDirSync, + }, + { + name: "install failure restores active", + configure: func(t *testing.T, activePath string, deps *recoveryOperationDeps) { + originalMove := deps.installMove + resolvedActive := canonicalRecoveryTestPath(t, activePath) + deps.installMove = func(oldPath, newPath string) error { + if strings.Contains(filepath.Base(oldPath), ".backscroll-recover-") && newPath == resolvedActive { + return fmt.Errorf("injected install failure") + } + return originalMove(oldPath, newPath) + } + }, + wantPhase: phaseInstall, + }, + { + name: "second directory fsync leaves durable backup", + configure: func(t *testing.T, activePath string, deps *recoveryOperationDeps) { + calls := 0 + originalSyncDir := deps.syncDir + deps.syncDir = func(path string) error { + calls++ + if calls == 2 { + return fmt.Errorf("injected second directory fsync failure") + } + return originalSyncDir(path) + } + }, + wantPhase: phaseInstallSync, + }, + } { + t.Run(tt.name, func(t *testing.T) { + dir := t.TempDir() + activePath := filepath.Join(dir, "active.db") + fromPath := filepath.Join(dir, "stranded.db") + createRecoveryDB(t, activePath, []storage.IndexedMessage{{ + Ordinal: 0, Role: "user", Text: "active typed fault", UUID: "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", Timestamp: "2026-08-18T00:00:00Z", ContentType: "text", + }}) + createRecoveryDB(t, fromPath, []storage.IndexedMessage{{ + Ordinal: 0, Role: "assistant", Text: "stranded typed fault", UUID: "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb", Timestamp: "2026-08-18T00:01:00Z", ContentType: "text", + }}) + activeBefore := snapshotRecoveryDBFile(t, activePath) + deps := defaultRecoveryOperationDeps() + tt.configure(t, activePath, &deps) + + report, err := executeWithDeps(context.Background(), Options{ActivePath: activePath, FromPath: fromPath}, deps) + if err == nil { + t.Fatal("executeWithDeps succeeded; want injected typed replacement failure") + } + var replacementErr *replacementError + if !errors.As(err, &replacementErr) { + t.Fatalf("error %T %[1]v, want *replacementError", err) + } + if replacementErr.Phase != tt.wantPhase { + t.Fatalf("phase = %s, want %s", replacementErr.Phase, tt.wantPhase) + } + if tt.wantPhase == phaseInstallSync { + if report.BackupPath == "" || !replacementErr.RestorableBackup { + t.Fatalf("second fsync failure consumed backup: report=%+v err=%+v", report, replacementErr) + } + backup := snapshotRecoveryDBFile(t, report.BackupPath) + if !bytes.Equal(backup.main.data, activeBefore.main.data) { + t.Fatalf("durable backup after second fsync failure differs from original active") + } + } else if tt.wantPhase == phaseBackupMain || tt.wantPhase == phaseBackupFileSync || tt.wantPhase == phaseBackupDirSync || tt.wantPhase == phaseInstall { + assertRecoveryDBFileSnapshot(t, "active after restorative failure", activePath, activeBefore) + } + }) + } +} + +func TestRecoverReplacementStatesAreStructurallyTruthful(t *testing.T) { + for _, tt := range []struct { + name string + configure func(t *testing.T, activePath string, deps *recoveryOperationDeps) + wantPhase replacementPhase + wantState replacementState + wantRestorable bool + wantBackupVisible bool + wantBackupVerified bool + wantBackupUnknown bool + wantRestoredVerified bool + wantRestoredVisible bool + wantReplacementVisible bool + wantReplacementDurable bool + wantNoActiveMutation bool + wantInventory string + wantCleanupErr bool + wantMilestones ApplyFailureMilestones + }{ + { + name: "first backup failure does not advertise candidate as visible", + configure: func(t *testing.T, activePath string, deps *recoveryOperationDeps) { + resolvedActive := canonicalRecoveryTestPath(t, activePath) + deps.noClobberMove = func(oldPath, newPath string) error { + if oldPath == resolvedActive { + return fmt.Errorf("injected first backup failure") + } + return noClobberMove(oldPath, newPath) + } + }, + wantPhase: phaseBackupMain, + wantState: replacementStateNoActiveMutation, + wantNoActiveMutation: true, + wantInventory: "unchanged", + }, + { + name: "backup file fsync failure restores active only after verification", + configure: func(t *testing.T, activePath string, deps *recoveryOperationDeps) { + deps.syncFile = func(path string) error { + if strings.Contains(filepath.Base(path), ".backup-") { + return fmt.Errorf("injected backup file fsync failure") + } + return syncFile(path) + } + }, + wantPhase: phaseBackupFileSync, + wantState: replacementStateActiveRestoredDurableVerified, + wantRestoredVerified: true, + wantRestoredVisible: true, + wantInventory: "unchanged", + wantMilestones: ApplyFailureMilestones{BackupCreated: true, RestoredActiveVisible: true, RestoredActiveDurableVerified: true}, + }, + { + name: "first directory fsync failure restores active only after verification", + configure: func(t *testing.T, activePath string, deps *recoveryOperationDeps) { + calls := 0 + deps.syncDir = func(path string) error { + calls++ + if calls == 1 { + return fmt.Errorf("injected first directory fsync failure") + } + return syncDir(path) + } + }, + wantPhase: phaseBackupDirSync, + wantState: replacementStateActiveRestoredDurableVerified, + wantRestoredVerified: true, + wantRestoredVisible: true, + wantInventory: "unchanged", + wantMilestones: ApplyFailureMilestones{BackupCreated: true, BackupFileSynced: true, RestoredActiveVisible: true, RestoredActiveDurableVerified: true}, + }, + { + name: "install failure restores active only after verification", + configure: func(t *testing.T, activePath string, deps *recoveryOperationDeps) { + resolvedActive := canonicalRecoveryTestPath(t, activePath) + deps.installMove = func(oldPath, newPath string) error { + if strings.Contains(filepath.Base(oldPath), ".backscroll-recover-") && newPath == resolvedActive { + return fmt.Errorf("injected install failure") + } + return noClobberMove(oldPath, newPath) + } + }, + wantPhase: phaseInstall, + wantState: replacementStateActiveRestoredDurableVerified, + wantRestoredVerified: true, + wantRestoredVisible: true, + wantInventory: "unchanged", + wantMilestones: ApplyFailureMilestones{BackupCreated: true, BackupFileSynced: true, BackupDirectoryDurable: true, BackupIntegrityVerified: true, RestoredActiveVisible: true, RestoredActiveDurableVerified: true}, + }, + { + name: "second directory fsync failure reports visible replacement and restorable verified backup", + configure: func(t *testing.T, activePath string, deps *recoveryOperationDeps) { + calls := 0 + deps.syncDir = func(path string) error { + calls++ + if calls == 2 { + return fmt.Errorf("injected second directory fsync failure") + } + return syncDir(path) + } + }, + wantPhase: phaseInstallSync, + wantState: replacementStateReplacementVisible, + wantRestorable: true, + wantBackupVisible: true, + wantBackupVerified: true, + wantReplacementVisible: true, + wantInventory: "replacement-and-backup", + wantMilestones: ApplyFailureMilestones{BackupCreated: true, BackupFileSynced: true, BackupDirectoryDurable: true, BackupIntegrityVerified: true, ReplacementVisible: true}, + }, + { + name: "pre-install final backup mismatch restores active; backup uncertainty remains historical only", + configure: func(t *testing.T, activePath string, deps *recoveryOperationDeps) { + original := deps.fingerprintSet + backupFingerprints := 0 + deps.fingerprintSet = func(path string) (sqliteFileSetSnapshot, error) { + snapshot, err := original(path) + if err == nil && strings.Contains(filepath.Base(path), ".backup-") { + backupFingerprints++ + if backupFingerprints == 2 { + snapshot.main.sha256 = "sha256:injected-mismatch" + } + } + return snapshot, err + } + }, + wantPhase: phasePostBackupVerify, + wantState: replacementStateActiveRestoredDurableVerified, + wantRestoredVerified: true, + wantRestoredVisible: true, + wantInventory: "unchanged", + wantMilestones: ApplyFailureMilestones{BackupCreated: true, BackupFileSynced: true, BackupDirectoryDurable: true, RestoredActiveVisible: true, RestoredActiveDurableVerified: true}, + }, + { + name: "post-install backup mismatch emits replacement durable and not restorable verified", + configure: func(t *testing.T, activePath string, deps *recoveryOperationDeps) { + original := deps.fingerprintSet + backupFingerprints := 0 + deps.fingerprintSet = func(path string) (sqliteFileSetSnapshot, error) { + snapshot, err := original(path) + if err == nil && strings.Contains(filepath.Base(path), ".backup-") { + backupFingerprints++ + if backupFingerprints == 3 { + snapshot.main.sha256 = "sha256:post-install-backup-mismatch" + } + } + return snapshot, err + } + }, + wantPhase: phaseInstallSync, + wantState: replacementStateReplacementDurable, + wantBackupVisible: true, + wantBackupUnknown: true, + wantReplacementVisible: true, + wantReplacementDurable: true, + wantInventory: "replacement-and-backup", + wantMilestones: ApplyFailureMilestones{BackupCreated: true, BackupFileSynced: true, BackupDirectoryDurable: true, BackupIntegrityVerified: true, ReplacementVisible: true, ReplacementDurable: true}, + }, + { + name: "restore move failure is split unknown and preserves verified backup path", + configure: func(t *testing.T, activePath string, deps *recoveryOperationDeps) { + resolvedActive := canonicalRecoveryTestPath(t, activePath) + deps.installMove = func(oldPath, newPath string) error { + if strings.Contains(filepath.Base(oldPath), ".backscroll-recover-") && newPath == resolvedActive { + return fmt.Errorf("injected install before restore move failure") + } + return noClobberMove(oldPath, newPath) + } + deps.restoreMove = func(oldPath, newPath string) error { + if newPath == resolvedActive { + return fmt.Errorf("injected restore move failure") + } + return noClobberMove(oldPath, newPath) + } + }, + wantPhase: phaseInstall, + wantState: replacementStateSplitUnknown, + wantRestorable: true, + wantBackupVisible: true, + wantBackupVerified: true, + wantInventory: "backup-only", + wantMilestones: ApplyFailureMilestones{BackupCreated: true, BackupFileSynced: true, BackupDirectoryDurable: true, BackupIntegrityVerified: true}, + }, + { + name: "restore fsync failure reports visible restored active with unknown integrity", + configure: func(t *testing.T, activePath string, deps *recoveryOperationDeps) { + resolvedActive := canonicalRecoveryTestPath(t, activePath) + restoredMain := false + deps.installMove = func(oldPath, newPath string) error { + if strings.Contains(filepath.Base(oldPath), ".backscroll-recover-") && newPath == resolvedActive { + return fmt.Errorf("injected install before restore fsync failure") + } + return noClobberMove(oldPath, newPath) + } + deps.restoreMove = func(oldPath, newPath string) error { + if err := noClobberMove(oldPath, newPath); err != nil { + return err + } + if newPath == resolvedActive { + restoredMain = true + } + return nil + } + deps.syncFile = func(path string) error { + if restoredMain && path == resolvedActive { + return fmt.Errorf("injected restored active fsync failure") + } + return syncFile(path) + } + }, + wantPhase: phaseInstall, + wantState: replacementStateActiveRestoredVisibleUnknown, + wantRestoredVisible: true, + wantRestoredVerified: false, + wantInventory: "unchanged", + wantMilestones: ApplyFailureMilestones{BackupCreated: true, BackupFileSynced: true, BackupDirectoryDurable: true, BackupIntegrityVerified: true, RestoredActiveVisible: true}, + }, + { + name: "restore fingerprint failure reports visible restored active with unknown integrity", + configure: func(t *testing.T, activePath string, deps *recoveryOperationDeps) { + resolvedActive := canonicalRecoveryTestPath(t, activePath) + restoredMain := false + deps.installMove = func(oldPath, newPath string) error { + if strings.Contains(filepath.Base(oldPath), ".backscroll-recover-") && newPath == resolvedActive { + return fmt.Errorf("injected install before restore fingerprint failure") + } + return noClobberMove(oldPath, newPath) + } + deps.restoreMove = func(oldPath, newPath string) error { + if err := noClobberMove(oldPath, newPath); err != nil { + return err + } + if newPath == resolvedActive { + restoredMain = true + } + return nil + } + original := deps.fingerprintSet + deps.fingerprintSet = func(path string) (sqliteFileSetSnapshot, error) { + snapshot, err := original(path) + if err == nil && restoredMain && path == resolvedActive { + snapshot.main.sha256 = "sha256:restored-fingerprint-mismatch" + } + return snapshot, err + } + }, + wantPhase: phaseInstall, + wantState: replacementStateActiveRestoredVisibleUnknown, + wantRestoredVisible: true, + wantInventory: "unchanged", + wantMilestones: ApplyFailureMilestones{BackupCreated: true, BackupFileSynced: true, BackupDirectoryDurable: true, BackupIntegrityVerified: true, RestoredActiveVisible: true}, + }, + { + name: "unsupported no-clobber reports no active mutation and cleanup failure without losing primary error", + configure: func(t *testing.T, activePath string, deps *recoveryOperationDeps) { + resolvedActive := canonicalRecoveryTestPath(t, activePath) + deps.noClobberMove = func(oldPath, newPath string) error { + if oldPath == resolvedActive { + return noClobberUnsupportedError{oldPath: oldPath, newPath: newPath} + } + return noClobberMove(oldPath, newPath) + } + originalCleanup := deps.cleanupFileSet + deps.cleanupFileSet = func(path string) error { + if err := originalCleanup(path); err != nil { + return err + } + return fmt.Errorf("injected cleanup visibility failure") + } + }, + wantPhase: phaseBackupMain, + wantState: replacementStateNoActiveMutation, + wantNoActiveMutation: true, + wantCleanupErr: true, + wantInventory: "unchanged", + }, + } { + t.Run(tt.name, func(t *testing.T) { + dir := t.TempDir() + activePath := filepath.Join(dir, "active.db") + fromPath := filepath.Join(dir, "stranded.db") + createRecoveryDB(t, activePath, []storage.IndexedMessage{{Ordinal: 0, Role: "user", Text: "active truthful state", UUID: "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", Timestamp: "2026-08-18T00:00:00Z", ContentType: "text"}}) + createRecoveryDB(t, fromPath, []storage.IndexedMessage{{Ordinal: 0, Role: "assistant", Text: "stranded truthful state", UUID: "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb", Timestamp: "2026-08-18T00:01:00Z", ContentType: "text"}}) + activeBefore := snapshotRecoveryDBFile(t, activePath) + beforeInventory := recoveryDirectoryInventory(t, dir) + deps := defaultRecoveryOperationDeps() + deps.randomHex = deterministicRecoveryHex() + tt.configure(t, activePath, &deps) + + report, err := executeWithDeps(context.Background(), Options{ActivePath: activePath, FromPath: fromPath}, deps) + if err == nil { + t.Fatal("executeWithDeps succeeded; want injected replacement failure") + } + var replacementErr *replacementError + if !errors.As(err, &replacementErr) { + t.Fatalf("error %T %[1]v, want *replacementError", err) + } + if replacementErr.Phase != tt.wantPhase || replacementErr.State != tt.wantState { + t.Fatalf("phase/state = %s/%s, want %s/%s; err=%v", replacementErr.Phase, replacementErr.State, tt.wantPhase, tt.wantState, err) + } + assertReplacementErrorPaths(t, replacementErr, activePath) + if replacementErr.RestorableBackup != tt.wantRestorable { + t.Fatalf("RestorableBackup = %v, want %v; err=%+v", replacementErr.RestorableBackup, tt.wantRestorable, replacementErr) + } + if tt.wantRestorable { + if replacementErr.CurrentRestorablePath == "" || replacementErr.CurrentRestorablePath != replacementErr.RestorableBackupPath { + t.Fatalf("CurrentRestorablePath/RestorableBackupPath = %q/%q, want matching current path", replacementErr.CurrentRestorablePath, replacementErr.RestorableBackupPath) + } + } else if replacementErr.CurrentRestorablePath != "" || replacementErr.RestorableBackupPath != "" { + t.Fatalf("restorable paths = %q/%q, want empty when not currently restorable", replacementErr.CurrentRestorablePath, replacementErr.RestorableBackupPath) + } + if replacementErr.BackupVisible != tt.wantBackupVisible || replacementErr.BackupIntegrityVerified != tt.wantBackupVerified || replacementErr.BackupIntegrityUnknown != tt.wantBackupUnknown { + t.Fatalf("backup fields visible/verified/unknown = %v/%v/%v, want %v/%v/%v; err=%+v", replacementErr.BackupVisible, replacementErr.BackupIntegrityVerified, replacementErr.BackupIntegrityUnknown, tt.wantBackupVisible, tt.wantBackupVerified, tt.wantBackupUnknown, replacementErr) + } + if replacementErr.RestoredActiveVisible != tt.wantRestoredVisible || replacementErr.RestoredActiveDurableVerified != tt.wantRestoredVerified { + t.Fatalf("restore fields visible/durableVerified = %v/%v, want %v/%v; err=%+v", replacementErr.RestoredActiveVisible, replacementErr.RestoredActiveDurableVerified, tt.wantRestoredVisible, tt.wantRestoredVerified, replacementErr) + } + if replacementErr.ReplacementVisible != tt.wantReplacementVisible || replacementErr.ReplacementDurable != tt.wantReplacementDurable { + t.Fatalf("replacement fields visible/durable = %v/%v, want %v/%v; err=%+v", replacementErr.ReplacementVisible, replacementErr.ReplacementDurable, tt.wantReplacementVisible, tt.wantReplacementDurable, replacementErr) + } + if replacementErr.NoActiveMutation != tt.wantNoActiveMutation { + t.Fatalf("NoActiveMutation = %v, want %v", replacementErr.NoActiveMutation, tt.wantNoActiveMutation) + } + if (replacementErr.CleanupErr != nil) != tt.wantCleanupErr { + t.Fatalf("CleanupErr = %v, want present=%v", replacementErr.CleanupErr, tt.wantCleanupErr) + } + if !reflect.DeepEqual(replacementErr.Milestones, tt.wantMilestones) { + t.Fatalf("Milestones = %+v, want %+v", replacementErr.Milestones, tt.wantMilestones) + } + assertNoRecoveryTemps(t, dir) + assertRecoveryFaultInventory(t, tt.wantInventory, dir, activePath, fromPath, replacementErr.BackupPath, activeBefore, beforeInventory, report.BackupPath) + }) + } +} + +func TestRecoverPreReplacementCloseAndCleanupErrorsAreTyped(t *testing.T) { + for _, tt := range []struct { + name string + configure func(t *testing.T, activePath string, deps *recoveryOperationDeps) + wantPhase replacementPhase + wantClose bool + wantCleanup bool + }{ + { + name: "reservation close failure closes before active mutation and removes temp", + configure: func(t *testing.T, activePath string, deps *recoveryOperationDeps) { + originalClose := deps.closeReservation + deps.closeReservation = func(r *activeReservation) error { + if err := originalClose(r); err != nil { + return err + } + return fmt.Errorf("injected reservation close failure") + } + }, + wantPhase: phaseReservationClose, + wantClose: true, + }, + { + name: "pre replacement cleanup failure is joined without overwriting primary error", + configure: func(t *testing.T, activePath string, deps *recoveryOperationDeps) { + resolvedActive := canonicalRecoveryTestPath(t, activePath) + originalFingerprint := deps.fingerprintSet + activeFingerprints := 0 + deps.fingerprintSet = func(path string) (sqliteFileSetSnapshot, error) { + snapshot, err := originalFingerprint(path) + if err == nil && path == resolvedActive { + activeFingerprints++ + if activeFingerprints == 3 { + snapshot.main.sha256 = "sha256:pre-replacement-mismatch" + } + } + return snapshot, err + } + originalCleanup := deps.cleanupFileSet + deps.cleanupFileSet = func(path string) error { + _ = originalCleanup(path) + return fmt.Errorf("injected pre-replacement cleanup failure") + } + }, + wantPhase: phasePreBackupValidate, + wantCleanup: true, + }, + } { + t.Run(tt.name, func(t *testing.T) { + dir := t.TempDir() + activePath := filepath.Join(dir, "active.db") + fromPath := filepath.Join(dir, "stranded.db") + createRecoveryDB(t, activePath, []storage.IndexedMessage{{Ordinal: 0, Role: "user", Text: "active pre replacement", UUID: "cccccccc-cccc-4ccc-8ccc-cccccccccccc", ContentType: "text"}}) + createRecoveryDB(t, fromPath, []storage.IndexedMessage{{Ordinal: 0, Role: "assistant", Text: "stranded pre replacement", UUID: "dddddddd-dddd-4ddd-8ddd-dddddddddddd", ContentType: "text"}}) + before := recoveryDirectoryInventory(t, dir) + deps := defaultRecoveryOperationDeps() + deps.randomHex = deterministicRecoveryHex() + tt.configure(t, activePath, &deps) + + _, err := executeWithDeps(context.Background(), Options{ActivePath: activePath, FromPath: fromPath}, deps) + if err == nil { + t.Fatal("executeWithDeps succeeded; want pre-replacement failure") + } + var preErr *preReplacementError + if !errors.As(err, &preErr) { + t.Fatalf("error %T %[1]v, want *preReplacementError", err) + } + if preErr.Phase != tt.wantPhase || preErr.State != replacementStateNoActiveMutation { + t.Fatalf("phase/state = %s/%s, want %s/%s", preErr.Phase, preErr.State, tt.wantPhase, replacementStateNoActiveMutation) + } + if preErr.ActivePath != canonicalRecoveryTestPath(t, activePath) || preErr.TempPath == "" { + t.Fatalf("pre-replacement paths = active %q temp %q", preErr.ActivePath, preErr.TempPath) + } + if (preErr.CloseErr != nil) != tt.wantClose || (preErr.CleanupErr != nil) != tt.wantCleanup { + t.Fatalf("close/cleanup errs = %v/%v, want present %v/%v", preErr.CloseErr, preErr.CleanupErr, tt.wantClose, tt.wantCleanup) + } + assertNoRecoveryTemps(t, dir) + if after := recoveryDirectoryInventory(t, dir); !reflect.DeepEqual(after, before) { + t.Fatalf("pre-replacement failure mutated inventory\nbefore: %s\nafter: %s", describeRecoveryInventory(before), describeRecoveryInventory(after)) + } + }) + } +} + +func TestRecoverPreReplacementFailureJoinsPrimaryCleanupAndClose(t *testing.T) { + dir := t.TempDir() + activePath := filepath.Join(dir, "active.db") + fromPath := filepath.Join(dir, "stranded.db") + createRecoveryDB(t, activePath, []storage.IndexedMessage{{Ordinal: 0, Role: "user", Text: "active joined errors", UUID: "90909090-9090-4090-8090-909090909090", ContentType: "text"}}) + createRecoveryDB(t, fromPath, []storage.IndexedMessage{{Ordinal: 0, Role: "assistant", Text: "stranded joined errors", UUID: "91919191-9191-4191-8191-919191919191", ContentType: "text"}}) + primaryErr := errors.New("injected primary pre-replacement validation failure") + cleanupErr := errors.New("injected cleanup failure") + closeErr := errors.New("injected reservation close failure") + resolvedActive := canonicalRecoveryTestPath(t, activePath) + deps := defaultRecoveryOperationDeps() + deps.randomHex = deterministicRecoveryHex() + originalFingerprint := deps.fingerprintSet + activeFingerprints := 0 + deps.fingerprintSet = func(path string) (sqliteFileSetSnapshot, error) { + if path == resolvedActive { + activeFingerprints++ + if activeFingerprints == 3 { + return sqliteFileSetSnapshot{}, primaryErr + } + } + return originalFingerprint(path) + } + originalCleanup := deps.cleanupFileSet + deps.cleanupFileSet = func(path string) error { + _ = originalCleanup(path) + return cleanupErr + } + originalClose := deps.closeReservation + deps.closeReservation = func(r *activeReservation) error { + return errors.Join(originalClose(r), closeErr) + } + + _, err := executeWithDeps(context.Background(), Options{ActivePath: activePath, FromPath: fromPath}, deps) + if err == nil { + t.Fatal("executeWithDeps succeeded; want joined primary cleanup and close errors") + } + for _, want := range []error{primaryErr, cleanupErr, closeErr} { + if !errors.Is(err, want) { + t.Fatalf("error %v does not contain %v", err, want) + } + } + var failure *ApplyFailure + if !errors.As(err, &failure) || failure.CleanupErr == nil { + t.Fatalf("error %T %[1]v, want ApplyFailure with cleanup", err) + } +} + +func TestRecoverNonDryRunPlanningDiagnosticsAreStructured(t *testing.T) { + dir := t.TempDir() + activePath := filepath.Join(dir, "active.db") + fromPath := filepath.Join(dir, "stranded.db") + sharedUUID := "01010101-0101-4101-8101-010101010101" + createRecoveryDB(t, activePath, []storage.IndexedMessage{{Ordinal: 0, Role: "user", Text: "active conflict", UUID: sharedUUID, ContentType: "text"}}) + createRecoveryDB(t, fromPath, []storage.IndexedMessage{{Ordinal: 0, Role: "user", Text: "stranded conflict", UUID: sharedUUID, ContentType: "text"}}) + + report, err := Execute(context.Background(), Options{ActivePath: activePath, FromPath: fromPath}) + if err == nil { + t.Fatal("Execute apply conflict succeeded; want structured planning failure") + } + var failure *ApplyFailure + if !errors.As(err, &failure) { + t.Fatalf("error %T %[1]v, want *ApplyFailure", err) + } + if failure.Phase != phasePlanning || failure.State != replacementStateNoActiveMutation || !failure.NoActiveMutation { + t.Fatalf("failure phase/state/no-active-mutation = %s/%s/%v", failure.Phase, failure.State, failure.NoActiveMutation) + } + if failure.ActivePath != canonicalRecoveryTestPath(t, activePath) || failure.FromPath != canonicalRecoveryTestPath(t, fromPath) || failure.TempPath != "" || failure.BackupPath != "" { + t.Fatalf("failure paths active=%q from=%q backup=%q temp=%q", failure.ActivePath, failure.FromPath, failure.BackupPath, failure.TempPath) + } + if len(report.Conflicts) == 0 || report.FinalCount != 0 { + t.Fatalf("report conflicts/final = %d/%d, want diagnostics and no final records", len(report.Conflicts), report.FinalCount) + } +} + +func TestRecoverNonDryRunMissingPathsAreStructured(t *testing.T) { + dir := t.TempDir() + activePath := filepath.Join(dir, "active.db") + fromPath := filepath.Join(dir, "stranded.db") + + for _, tt := range []struct { + name string + opts Options + wantActive string + wantFrom string + }{ + { + name: "missing active", + opts: Options{FromPath: fromPath}, + wantFrom: knownPathDescription(fromPath), + }, + { + name: "missing from", + opts: Options{ActivePath: activePath}, + wantActive: knownPathDescription(activePath), + }, + } { + t.Run(tt.name, func(t *testing.T) { + _, err := Execute(context.Background(), tt.opts) + if err == nil { + t.Fatal("Execute succeeded; want structured missing path failure") + } + var failure *ApplyFailure + if !errors.As(err, &failure) { + t.Fatalf("error %T %[1]v, want *ApplyFailure", err) + } + if failure.Phase != phaseSourceRead || failure.State != replacementStateNoActiveMutation || !failure.NoActiveMutation { + t.Fatalf("failure phase/state/no-active-mutation = %s/%s/%v", failure.Phase, failure.State, failure.NoActiveMutation) + } + if failure.ActivePath != tt.wantActive || failure.FromPath != tt.wantFrom { + t.Fatalf("failure paths active=%q from=%q, want active=%q from=%q", failure.ActivePath, failure.FromPath, tt.wantActive, tt.wantFrom) + } + }) + } +} + +func TestRecoverDryRunMissingPathsRemainValidationErrors(t *testing.T) { + _, err := Execute(context.Background(), Options{DryRun: true}) + if err == nil { + t.Fatal("dry-run Execute succeeded; want raw validation error") + } + var failure *ApplyFailure + if errors.As(err, &failure) { + t.Fatalf("dry-run validation error %T %[1]v unexpectedly matches *ApplyFailure", err) + } +} + +func TestRecoverDryRunInputValidationErrorsStayPlain(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + activePath := filepath.Join(dir, "active.db") + fromPath := filepath.Join(dir, "from.db") + createRecoveryDB(t, activePath, []storage.IndexedMessage{{Ordinal: 0, Role: "user", Text: "active dry-run validation", UUID: "11111111-1111-4111-8111-111111111111", ContentType: "text"}}) + createRecoveryDB(t, fromPath, []storage.IndexedMessage{{Ordinal: 0, Role: "assistant", Text: "from dry-run validation", UUID: "22222222-2222-4222-8222-222222222222", ContentType: "text"}}) + brokenLink := filepath.Join(dir, "broken-link.db") + if err := os.Symlink(filepath.Join(dir, "missing-target.db"), brokenLink); err != nil { + t.Skipf("symlink unsupported: %v", err) + } + + cases := []struct { + name string + opts Options + }{ + {name: "missing_from", opts: Options{ActivePath: activePath, FromPath: "", DryRun: true}}, + {name: "broken_active", opts: Options{ActivePath: brokenLink, FromPath: fromPath, DryRun: true}}, + {name: "broken_from", opts: Options{ActivePath: activePath, FromPath: brokenLink, DryRun: true}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, err := Execute(ctx, tc.opts) + if err == nil { + t.Fatalf("Execute(%+v) succeeded; want dry-run validation error", tc.opts) + } + var failure *ApplyFailure + if errors.As(err, &failure) { + t.Fatalf("dry-run error %v was ApplyFailure %+v; want plain validation error", err, failure) + } + }) + } +} + +func TestRecoverDryRunConflictReportsDiagnosticsWithoutApplyFailure(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + activePath := filepath.Join(dir, "active.db") + fromPath := filepath.Join(dir, "from.db") + sharedUUID := "33333333-3333-4333-8333-333333333333" + createRecoveryDB(t, activePath, []storage.IndexedMessage{{Ordinal: 0, Role: "user", Text: "active conflicting dry run", UUID: sharedUUID, ContentType: "text"}}) + createRecoveryDB(t, fromPath, []storage.IndexedMessage{{Ordinal: 0, Role: "assistant", Text: "from conflicting dry run", UUID: sharedUUID, ContentType: "text"}}) + + report, err := Execute(ctx, Options{ActivePath: activePath, FromPath: fromPath, DryRun: true}) + if err == nil { + t.Fatal("dry-run conflict succeeded; want diagnostic error") + } + var failure *ApplyFailure + if errors.As(err, &failure) { + t.Fatalf("dry-run conflict returned ApplyFailure %+v; want plain diagnostic error", failure) + } + if len(report.Conflicts) == 0 || report.Conflicts[0].Code != compat.CodeRecoveryConflict { + t.Fatalf("dry-run conflict report diagnostics = %+v, want recovery conflict", report.Conflicts) + } + if report.FinalCount != 0 { + t.Fatalf("dry-run conflict final count = %d, want no applicable final count", report.FinalCount) + } +} + +func TestApplyFailurePublicErrorDetailsExposeRestorableBackup(t *testing.T) { + dir := t.TempDir() + activePath := filepath.Join(dir, "active.db") + fromPath := filepath.Join(dir, "stranded.db") + createRecoveryDB(t, activePath, []storage.IndexedMessage{{Ordinal: 0, Role: "user", Text: "active before public restorable failure", UUID: "11111111-1111-4111-8111-111111111111", ContentType: "text"}}) + createRecoveryDB(t, fromPath, []storage.IndexedMessage{{Ordinal: 0, Role: "assistant", Text: "stranded public restorable failure", UUID: "22222222-2222-4222-8222-222222222222", ContentType: "text"}}) + activeBefore := snapshotRecoveryDBFile(t, activePath) + + injectedSidecar := activePath + "-wal" + sidecarDone := make(chan error, 1) + stopSidecar := make(chan struct{}) + go func() { + for { + select { + case <-stopSidecar: + sidecarDone <- fmt.Errorf("active namespace was never opened for replacement") + return + default: + } + if _, err := os.Lstat(activePath); os.IsNotExist(err) { + sidecarDone <- os.WriteFile(injectedSidecar, []byte("late unsafe wal sidecar"), 0o600) + return + } + time.Sleep(100 * time.Microsecond) + } + }() + + report, err := Execute(context.Background(), Options{ActivePath: activePath, FromPath: fromPath}) + close(stopSidecar) + if sidecarErr := <-sidecarDone; sidecarErr != nil { + t.Fatalf("inject late active sidecar: %v", sidecarErr) + } + if err == nil { + t.Fatalf("Execute succeeded with late active sidecar; report=%+v", report) + } + var failure *ApplyFailure + if !errors.As(err, &failure) { + t.Fatalf("error %T %[1]v, want *ApplyFailure from exported Execute", err) + } + if !errors.Is(err, storage.ErrImmutableReadOnlyWALUnsafe) { + t.Fatalf("error %v does not preserve unsafe WAL sentinel", err) + } + wantActive := canonicalRecoveryTestPath(t, activePath) + wantFrom := canonicalRecoveryTestPath(t, fromPath) + if failure.ActivePath != wantActive || failure.FromPath != wantFrom || failure.TempPath == "" { + t.Fatalf("failure paths active=%q from=%q temp=%q, want canonical active/from and temp", failure.ActivePath, failure.FromPath, failure.TempPath) + } + if failure.Phase != phaseInstall || failure.State != replacementStateSplitUnknown { + t.Fatalf("failure phase/state = %s/%s, want %s/%s", failure.Phase, failure.State, phaseInstall, replacementStateSplitUnknown) + } + if !failure.RestorableBackup || failure.RestorableBackupPath == "" || failure.RestorableBackupPath != failure.BackupPath { + t.Fatalf("restorable backup fields = restorable=%v path=%q backup=%q", failure.RestorableBackup, failure.RestorableBackupPath, failure.BackupPath) + } + if !failure.BackupVisible || !failure.BackupIntegrityVerified || failure.ActiveRestored || failure.RestoredActiveVisible { + t.Fatalf("replacement state fields = backupVisible=%v backupVerified=%v activeRestored=%v restoredVisible=%v", failure.BackupVisible, failure.BackupIntegrityVerified, failure.ActiveRestored, failure.RestoredActiveVisible) + } + if failure.Cause == nil || failure.RestoreErr == nil { + t.Fatalf("failure cause/restore = %v/%v, want both populated by Execute", failure.Cause, failure.RestoreErr) + } + if got := failure.Unwrap(); len(got) != 2 || got[0] != failure.Cause || got[1] != failure.RestoreErr { + t.Fatalf("ApplyFailure.Unwrap() = %#v, want cause then restore error", got) + } + if path, ok := RestorableBackupPath(fmt.Errorf("outer: %w", err)); !ok || path != failure.RestorableBackupPath { + t.Fatalf("RestorableBackupPath wrapped Execute failure = %q, %v; want %q", path, ok, failure.RestorableBackupPath) + } + if backup := snapshotRecoveryDBFile(t, failure.RestorableBackupPath); !bytes.Equal(backup.main.data, activeBefore.main.data) { + t.Fatalf("restorable backup differs from original active bytes") + } + if _, statErr := os.Lstat(activePath); !os.IsNotExist(statErr) { + t.Fatalf("active path state after split failure stat=%v, want missing main with restorable backup", statErr) + } + if _, statErr := os.Lstat(injectedSidecar); statErr != nil { + t.Fatalf("injected sidecar no longer demonstrates failed active namespace: %v", statErr) + } +} + +func TestRecoverPathCanonicalizationFailureIsStructured(t *testing.T) { + dir := t.TempDir() + + for _, tt := range []struct { + name string + setup func(t *testing.T) (activePath, fromPath string) + wantActive func(t *testing.T, path string) string + wantFrom func(t *testing.T, path string) string + }{ + { + name: "active symlink failure", + setup: func(t *testing.T) (string, string) { + activePath := filepath.Join(dir, "broken-active-link.db") + fromPath := filepath.Join(dir, "stranded.db") + createRecoveryDB(t, fromPath, []storage.IndexedMessage{{Ordinal: 0, Role: "assistant", Text: "stranded", UUID: "07070707-0707-4707-8707-070707070707", ContentType: "text"}}) + if err := os.Symlink(filepath.Join(dir, "missing-active-target.db"), activePath); err != nil { + t.Skipf("symlink unsupported: %v", err) + } + return activePath, fromPath + }, + wantActive: func(t *testing.T, path string) string { return knownPathDescription(path) }, + wantFrom: func(t *testing.T, path string) string { return knownPathDescription(path) }, + }, + { + name: "from symlink failure", + setup: func(t *testing.T) (string, string) { + activePath := filepath.Join(dir, "active.db") + fromPath := filepath.Join(dir, "broken-from-link.db") + createRecoveryDB(t, activePath, []storage.IndexedMessage{{Ordinal: 0, Role: "user", Text: "active", UUID: "08080808-0808-4808-8808-080808080808", ContentType: "text"}}) + if err := os.Symlink(filepath.Join(dir, "missing-from-target.db"), fromPath); err != nil { + t.Skipf("symlink unsupported: %v", err) + } + return activePath, fromPath + }, + wantActive: canonicalRecoveryTestPath, + wantFrom: func(t *testing.T, path string) string { return knownPathDescription(path) }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + activePath, fromPath := tt.setup(t) + _, err := Execute(context.Background(), Options{ActivePath: activePath, FromPath: fromPath}) + if err == nil { + t.Fatal("Execute succeeded; want structured path canonicalization failure") + } + var failure *ApplyFailure + if !errors.As(err, &failure) { + t.Fatalf("error %T %[1]v, want *ApplyFailure", err) + } + if failure.Phase != phaseSourceRead || failure.State != replacementStateNoActiveMutation { + t.Fatalf("failure phase/state = %s/%s", failure.Phase, failure.State) + } + if failure.ActivePath != tt.wantActive(t, activePath) || failure.FromPath != tt.wantFrom(t, fromPath) { + t.Fatalf("failure paths active=%q from=%q", failure.ActivePath, failure.FromPath) + } + if !strings.Contains(err.Error(), "resolve database symlink") { + t.Fatalf("error = %v, want symlink path details", err) + } + }) + } +} + +func TestRecoverInitialApplyFailurePreservesCauseAs(t *testing.T) { + type sentinelError struct{ error } + cause := &sentinelError{error: errors.New("sentinel initial failure")} + err := initialApplyFailure(phaseSourceRead, "/active.db", "/from.db", cause) + var failure *ApplyFailure + if !errors.As(err, &failure) { + t.Fatalf("error %T %[1]v, want *ApplyFailure", err) + } + var gotCause *sentinelError + if !errors.As(err, &gotCause) || gotCause != cause { + t.Fatalf("errors.As did not preserve cause: %T %[1]v", err) + } +} + +func TestRecoverStrandedOpenFailureIsStructured(t *testing.T) { + dir := t.TempDir() + activePath := filepath.Join(dir, "active.db") + fromPath := filepath.Join(dir, "missing-stranded.db") + createRecoveryDB(t, activePath, []storage.IndexedMessage{{Ordinal: 0, Role: "user", Text: "active missing stranded", UUID: "06060606-0606-4606-8606-060606060606", ContentType: "text"}}) + + _, err := Execute(context.Background(), Options{ActivePath: activePath, FromPath: fromPath}) + if err == nil { + t.Fatal("Execute missing stranded source succeeded; want structured source-open failure") + } + var failure *ApplyFailure + if !errors.As(err, &failure) { + t.Fatalf("error %T %[1]v, want *ApplyFailure", err) + } + if failure.Phase != phaseSourceRead || failure.ActivePath != canonicalRecoveryTestPath(t, activePath) || failure.FromPath != knownPathDescription(fromPath) || failure.TempPath != "" { + t.Fatalf("failure phase/paths = %s active=%q from=%q temp=%q", failure.Phase, failure.ActivePath, failure.FromPath, failure.TempPath) + } +} + +func TestRecoverStrandedReadFailureCarriesCanonicalFromPath(t *testing.T) { + dir := t.TempDir() + activePath := filepath.Join(dir, "active.db") + fromPath := filepath.Join(dir, "stranded.db") + createRecoveryDB(t, activePath, []storage.IndexedMessage{{Ordinal: 0, Role: "user", Text: "active before stranded read fail", UUID: "0a0a0a0a-0a0a-4a0a-8a0a-0a0a0a0a0a0a", ContentType: "text"}}) + createRecoveryDB(t, fromPath, []storage.IndexedMessage{{Ordinal: 0, Role: "assistant", Text: "stranded read fail", UUID: "0b0b0b0b-0b0b-4b0b-8b0b-0b0b0b0b0b0b", ContentType: "text"}}) + wantErr := errors.New("injected stranded source read failure") + wantFromPath := canonicalRecoveryTestPath(t, fromPath) + deps := defaultRecoveryOperationDeps() + originalFingerprintSet := deps.fingerprintSet + deps.fingerprintSet = func(path string) (sqliteFileSetSnapshot, error) { + if path == wantFromPath { + return sqliteFileSetSnapshot{}, wantErr + } + return originalFingerprintSet(path) + } + + _, err := executeWithDeps(context.Background(), Options{ActivePath: activePath, FromPath: fromPath}, deps) + if err == nil || !errors.Is(err, wantErr) { + t.Fatalf("executeWithDeps error = %v, want stranded read sentinel", err) + } + var failure *ApplyFailure + if !errors.As(err, &failure) { + t.Fatalf("error %T %[1]v, want *ApplyFailure", err) + } + if failure.Phase != phaseSourceRead || failure.FromPath != wantFromPath { + t.Fatalf("failure phase/from = %s/%q, want source-read with canonical from %q", failure.Phase, failure.FromPath, wantFromPath) + } +} + +func TestRecoverActiveReadFailureIsStructured(t *testing.T) { + dir := t.TempDir() + activePath := filepath.Join(dir, "active.db") + fromPath := filepath.Join(dir, "stranded.db") + createRecoveryDB(t, activePath, []storage.IndexedMessage{{Ordinal: 0, Role: "user", Text: "active read fail", UUID: "02020202-0202-4202-8202-020202020202", ContentType: "text"}}) + createRecoveryDB(t, fromPath, []storage.IndexedMessage{{Ordinal: 0, Role: "assistant", Text: "stranded read fail", UUID: "03030303-0303-4303-8303-030303030303", ContentType: "text"}}) + wantErr := errors.New("injected active source read failure") + deps := defaultRecoveryOperationDeps() + deps.readReservedInput = func(context.Context, compat.Queryer) (compat.RecoveryInput, *compat.Diagnostic, error) { + return compat.RecoveryInput{}, nil, wantErr + } + + _, err := executeWithDeps(context.Background(), Options{ActivePath: activePath, FromPath: fromPath}, deps) + if err == nil || !errors.Is(err, wantErr) { + t.Fatalf("executeWithDeps error = %v, want source read sentinel", err) + } + var failure *ApplyFailure + if !errors.As(err, &failure) || failure.Phase != phaseSourceRead || failure.ActivePath != canonicalRecoveryTestPath(t, activePath) || failure.FromPath != canonicalRecoveryTestPath(t, fromPath) { + t.Fatalf("error = %T %[1]v, want source-read ApplyFailure for active with canonical paths", err) + } +} + +func TestRecoverDestinationConstructionCleanupLeakIsStructured(t *testing.T) { + dir := t.TempDir() + activePath := filepath.Join(dir, "active.db") + fromPath := filepath.Join(dir, "stranded.db") + createRecoveryDB(t, activePath, []storage.IndexedMessage{{Ordinal: 0, Role: "user", Text: "active destination leak", UUID: "04040404-0404-4404-8404-040404040404", ContentType: "text"}}) + createRecoveryDB(t, fromPath, []storage.IndexedMessage{{Ordinal: 0, Role: "assistant", Text: "stranded destination leak", UUID: "05050505-0505-4505-8505-050505050505", ContentType: "text"}}) + leakedPath := filepath.Join(dir, ".backscroll-recover-leaked.db") + if err := os.WriteFile(leakedPath, []byte("leaked temp sentinel"), 0o600); err != nil { + t.Fatalf("write leaked temp: %v", err) + } + causeErr := errors.New("injected destination construction failure") + cleanupErr := errors.New("injected destination cleanup failure") + deps := defaultRecoveryOperationDeps() + deps.createDestination = func(context.Context, string, compat.RecoveryPlan) (string, error) { + return leakedPath, &storage.RecoveryDestinationError{Path: leakedPath, Cause: causeErr, CleanupErr: cleanupErr} + } + + _, err := executeWithDeps(context.Background(), Options{ActivePath: activePath, FromPath: fromPath}, deps) + if err == nil || !errors.Is(err, causeErr) || !errors.Is(err, cleanupErr) { + t.Fatalf("executeWithDeps error = %v, want cause and cleanup", err) + } + var failure *ApplyFailure + if !errors.As(err, &failure) { + t.Fatalf("error %T %[1]v, want *ApplyFailure", err) + } + if failure.FromPath != canonicalRecoveryTestPath(t, fromPath) || failure.TempPath != leakedPath || failure.CleanupErr == nil || len(failure.CleanupErrors) != 1 { + t.Fatalf("FromPath/TempPath/CleanupErr/CleanupErrors = %q/%q/%v/%d, want canonical from path, leaked path, and structured cleanup", failure.FromPath, failure.TempPath, failure.CleanupErr, len(failure.CleanupErrors)) + } + if _, statErr := os.Lstat(leakedPath); statErr != nil { + t.Fatalf("leaked path %s missing: %v", leakedPath, statErr) + } +} + +func TestRecoverSameDirectoryFailureIsStructured(t *testing.T) { + dir := t.TempDir() + activePath := filepath.Join(dir, "active.db") + tempPath := filepath.Join(t.TempDir(), ".backscroll-recover-other.db") + _, err := replaceActiveWithBackup(context.Background(), activePath, tempPath, compat.RecoveryPlan{}, sqliteFileSetSnapshot{}, defaultRecoveryOperationDeps()) + if err == nil { + t.Fatal("replaceActiveWithBackup succeeded; want same-directory failure") + } + var failure *ApplyFailure + if !errors.As(err, &failure) { + t.Fatalf("error %T %[1]v, want *ApplyFailure", err) + } + if failure.Phase != phasePreBackupValidate || failure.State != replacementStateNoActiveMutation || failure.ActivePath != activePath || failure.TempPath != tempPath { + t.Fatalf("failure phase/state/paths = %s/%s/%q/%q", failure.Phase, failure.State, failure.ActivePath, failure.TempPath) + } +} + +func TestRecoverFingerprintJoinsReadAndCloseErrors(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "fingerprint.db") + if err := os.WriteFile(path, []byte("fingerprint bytes"), 0o600); err != nil { + t.Fatalf("write fingerprint source: %v", err) + } + pathInfo, err := os.Lstat(path) + if err != nil { + t.Fatalf("lstat fingerprint source: %v", err) + } + file, err := os.Open(path) + if err != nil { + t.Fatalf("open fingerprint source: %v", err) + } + readErr := errors.New("injected fingerprint read failure") + closeErr := errors.New("injected fingerprint close failure") + _, err = snapshotSQLiteOpenFile(path, pathInfo, &erroringSnapshotFile{file: file, readErr: readErr, closeErr: closeErr}) + if err == nil || !errors.Is(err, readErr) || !errors.Is(err, closeErr) { + t.Fatalf("snapshotSQLiteOpenFile error = %v, want read and close", err) + } +} + +type erroringSnapshotFile struct { + file *os.File + readErr error + closeErr error +} + +func (f *erroringSnapshotFile) Read([]byte) (int, error) { return 0, f.readErr } + +func (f *erroringSnapshotFile) Stat() (os.FileInfo, error) { return f.file.Stat() } + +func (f *erroringSnapshotFile) Close() error { + _ = f.file.Close() + return f.closeErr +} + +func TestRecoverRevalidatesConcurrentMutationBeforeInstall(t *testing.T) { + dir := t.TempDir() + activePath := filepath.Join(dir, "active.db") + fromPath := filepath.Join(dir, "stranded.db") + createRecoveryDB(t, activePath, []storage.IndexedMessage{{ + Ordinal: 0, Role: "user", Text: "active before concurrent mutation", UUID: "11111111-1111-4111-8111-111111111111", Timestamp: "2026-08-18T00:00:00Z", ContentType: "text", + }}) + createRecoveryDB(t, fromPath, []storage.IndexedMessage{{ + Ordinal: 0, Role: "assistant", Text: "stranded not installed after mutation", UUID: "22222222-2222-4222-8222-222222222222", Timestamp: "2026-08-18T00:01:00Z", ContentType: "text", + }}) + resolvedActivePath := canonicalRecoveryTestPath(t, activePath) + deps := defaultRecoveryOperationDeps() + originalMove := deps.noClobberMove + mutated := false + deps.noClobberMove = func(oldPath, newPath string) error { + if oldPath == resolvedActivePath && !mutated { + mutated = true + f, err := os.OpenFile(oldPath, os.O_WRONLY|os.O_APPEND, 0) + if err != nil { + return err + } + _, writeErr := f.WriteString("concurrent mutation") + closeErr := f.Close() + if writeErr != nil { + return writeErr + } + if closeErr != nil { + return closeErr + } + } + return originalMove(oldPath, newPath) + } + + _, err := executeWithDeps(context.Background(), Options{ActivePath: activePath, FromPath: fromPath}, deps) + if err == nil { + t.Fatal("recovery succeeded after concurrent mutation during backup; want fail closed") + } + var replacementErr *replacementError + if !errors.As(err, &replacementErr) || replacementErr.Phase != phasePostBackupVerify { + t.Fatalf("error = %T %[1]v, want post-backup verification replacement error", err) + } +} + +func TestRecoverActiveFingerprintRevalidationGaps(t *testing.T) { + for _, tt := range []struct { + name string + configure func(t *testing.T, activePath, fromPath string, deps *recoveryOperationDeps) + want string + }{ + { + name: "in-place mutation after reserved SQL read", + configure: func(t *testing.T, activePath, fromPath string, deps *recoveryOperationDeps) { + original := deps.readReservedInput + deps.readReservedInput = func(ctx context.Context, q compat.Queryer) (compat.RecoveryInput, *compat.Diagnostic, error) { + input, diag, err := original(ctx, q) + appendRecoveryTestBytes(t, activePath, "mutation after reserved read") + return input, diag, err + } + }, + want: "during recovery SQL read", + }, + { + name: "inode swap after reservation close", + configure: func(t *testing.T, activePath, fromPath string, deps *recoveryOperationDeps) { + original := deps.closeReservation + deps.closeReservation = func(r *activeReservation) error { + if err := original(r); err != nil { + return err + } + replaceRecoveryTestFile(t, activePath, []byte("not the planned sqlite main")) + return nil + } + }, + want: "after closing recovery reservation", + }, + { + name: "stranded in-place mutation during immutable read", + configure: func(t *testing.T, activePath, fromPath string, deps *recoveryOperationDeps) { + original := deps.fingerprintSet + mutated := false + resolvedFrom := canonicalRecoveryTestPath(t, fromPath) + deps.fingerprintSet = func(path string) (sqliteFileSetSnapshot, error) { + snapshot, err := original(path) + if err == nil && path == resolvedFrom && !mutated { + mutated = true + appendRecoveryTestBytes(t, fromPath, "stranded mutated after pre-read fingerprint") + } + return snapshot, err + } + }, + want: "immutable recovery source changed", + }, + } { + t.Run(tt.name, func(t *testing.T) { + dir := t.TempDir() + activePath := filepath.Join(dir, "active.db") + fromPath := filepath.Join(dir, "stranded.db") + createRecoveryDB(t, activePath, []storage.IndexedMessage{{Ordinal: 0, Role: "user", Text: "active gap", UUID: "10101010-1010-4010-8010-101010101010", ContentType: "text"}}) + createRecoveryDB(t, fromPath, []storage.IndexedMessage{{Ordinal: 0, Role: "assistant", Text: "stranded gap", UUID: "20202020-2020-4020-8020-202020202020", ContentType: "text"}}) + deps := defaultRecoveryOperationDeps() + tt.configure(t, activePath, fromPath, &deps) + + _, err := executeWithDeps(context.Background(), Options{ActivePath: activePath, FromPath: fromPath}, deps) + if err == nil || !strings.Contains(err.Error(), tt.want) { + t.Fatalf("executeWithDeps error = %v, want %q", err, tt.want) + } + }) + } +} + +func TestRecoverRejectsLateActiveSidecarAfterBackup(t *testing.T) { + dir := t.TempDir() + activePath := filepath.Join(dir, "active.db") + fromPath := filepath.Join(dir, "stranded.db") + createRecoveryDB(t, activePath, []storage.IndexedMessage{{Ordinal: 0, Role: "user", Text: "active late sidecar", UUID: "30303030-3030-4030-8030-303030303030", ContentType: "text"}}) + createRecoveryDB(t, fromPath, []storage.IndexedMessage{{Ordinal: 0, Role: "assistant", Text: "stranded late sidecar", UUID: "40404040-4040-4040-8040-404040404040", ContentType: "text"}}) + deps := defaultRecoveryOperationDeps() + original := deps.noClobberMove + resolvedActive := canonicalRecoveryTestPath(t, activePath) + deps.noClobberMove = func(oldPath, newPath string) error { + err := original(oldPath, newPath) + if err == nil && oldPath == resolvedActive { + if writeErr := os.WriteFile(resolvedActive+"-journal", []byte("late sidecar"), 0o600); writeErr != nil { + return writeErr + } + } + return err + } + + _, err := executeWithDeps(context.Background(), Options{ActivePath: activePath, FromPath: fromPath}, deps) + if err == nil || !strings.Contains(err.Error(), "unexpected active sidecar") || !strings.Contains(err.Error(), "-journal") { + t.Fatalf("executeWithDeps error = %v, want late sidecar rejection", err) + } + if _, statErr := os.Lstat(activePath); !os.IsNotExist(statErr) { + t.Fatalf("active main restored into unsafe namespace: stat err=%v", statErr) + } +} + +func TestRecoverSQLiteSidecarPolicy(t *testing.T) { + t.Run("stranded non-empty WAL rejected before immutable planning", func(t *testing.T) { + dir := t.TempDir() + activePath := filepath.Join(dir, "active.db") + fromPath := filepath.Join(dir, "stranded.db") + createRecoveryDB(t, activePath, []storage.IndexedMessage{{Ordinal: 0, Role: "user", Text: "active", UUID: "33333333-3333-4333-8333-333333333333", ContentType: "text"}}) + stranded := createRecoveryLiveWALDB(t, fromPath) + defer func() { _ = stranded.Close() }() + + _, err := Execute(context.Background(), Options{ActivePath: activePath, FromPath: fromPath}) + if err == nil || !errors.Is(err, storage.ErrImmutableReadOnlyWALUnsafe) { + t.Fatalf("Execute error = %v, want immutable WAL unsafe", err) + } + }) + + t.Run("rollback journal rejected", func(t *testing.T) { + dir := t.TempDir() + activePath := filepath.Join(dir, "active.db") + fromPath := filepath.Join(dir, "stranded.db") + createRecoveryDB(t, activePath, []storage.IndexedMessage{{Ordinal: 0, Role: "user", Text: "active", UUID: "44444444-4444-4444-8444-444444444444", ContentType: "text"}}) + createRecoveryDB(t, fromPath, []storage.IndexedMessage{{Ordinal: 0, Role: "assistant", Text: "stranded", UUID: "55555555-5555-4555-8555-555555555555", ContentType: "text"}}) + if err := os.WriteFile(activePath+"-journal", []byte("hot journal sentinel"), 0o600); err != nil { + t.Fatalf("write rollback journal: %v", err) + } + + _, err := Execute(context.Background(), Options{ActivePath: activePath, FromPath: fromPath}) + if err == nil || !strings.Contains(err.Error(), "sidecar namespace") || !strings.Contains(err.Error(), "-journal") { + t.Fatalf("Execute error = %v, want rollback journal namespace rejection", err) + } + }) + + t.Run("empty active shm rejected before replacement", func(t *testing.T) { + dir := t.TempDir() + activePath := filepath.Join(dir, "active.db") + fromPath := filepath.Join(dir, "stranded.db") + createRecoveryDB(t, activePath, []storage.IndexedMessage{{Ordinal: 0, Role: "user", Text: "active with stale shm", UUID: "66666666-6666-4666-8666-666666666666", ContentType: "text"}}) + createRecoveryDB(t, fromPath, []storage.IndexedMessage{{Ordinal: 0, Role: "assistant", Text: "stranded install", UUID: "77777777-7777-4777-8777-777777777777", ContentType: "text"}}) + if err := os.WriteFile(activePath+"-shm", nil, 0o600); err != nil { + t.Fatalf("write empty stale shm: %v", err) + } + + _, err := Execute(context.Background(), Options{ActivePath: activePath, FromPath: fromPath}) + if err == nil || !strings.Contains(err.Error(), "-shm") || !strings.Contains(err.Error(), "sidecar namespace") { + t.Fatalf("Execute error = %v, want empty shm namespace rejection", err) + } + }) + + t.Run("dangling stranded wal symlink rejected", func(t *testing.T) { + dir := t.TempDir() + activePath := filepath.Join(dir, "active.db") + fromPath := filepath.Join(dir, "stranded.db") + createRecoveryDB(t, activePath, []storage.IndexedMessage{{Ordinal: 0, Role: "user", Text: "active", UUID: "88888888-8888-4888-8888-888888888888", ContentType: "text"}}) + createRecoveryDB(t, fromPath, []storage.IndexedMessage{{Ordinal: 0, Role: "assistant", Text: "stranded", UUID: "99999999-9999-4999-8999-999999999999", ContentType: "text"}}) + if err := os.Symlink(filepath.Join(dir, "missing-wal-target"), fromPath+"-wal"); err != nil { + t.Fatalf("symlink dangling wal: %v", err) + } + + _, err := Execute(context.Background(), Options{ActivePath: activePath, FromPath: fromPath, DryRun: true}) + if err == nil || !errors.Is(err, storage.ErrImmutableReadOnlyWALUnsafe) || !strings.Contains(err.Error(), fromPath+"-wal") { + t.Fatalf("Execute error = %v, want dangling WAL symlink rejection", err) + } + }) +} + +func TestRecoverBackupNameCollisionRetriesWithoutClobber(t *testing.T) { + dir := t.TempDir() + activePath := filepath.Join(dir, "active.db") + fromPath := filepath.Join(dir, "stranded.db") + createRecoveryDB(t, activePath, []storage.IndexedMessage{{Ordinal: 0, Role: "user", Text: "active collision", UUID: "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", ContentType: "text"}}) + createRecoveryDB(t, fromPath, []storage.IndexedMessage{{Ordinal: 0, Role: "assistant", Text: "stranded collision", UUID: "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb", ContentType: "text"}}) + deps := defaultRecoveryOperationDeps() + originalMove := deps.noClobberMove + suffixes := []string{"000001", "000002"} + deps.randomHex = func(n int) (string, error) { + if len(suffixes) == 0 { + return "000002", nil + } + next := suffixes[0] + suffixes = suffixes[1:] + return next, nil + } + collided := false + resolvedActive := canonicalRecoveryTestPath(t, activePath) + deps.noClobberMove = func(oldPath, newPath string) error { + if oldPath == resolvedActive && !collided { + collided = true + return os.ErrExist + } + return originalMove(oldPath, newPath) + } + + report, err := executeWithDeps(context.Background(), Options{ActivePath: activePath, FromPath: fromPath}, deps) + if err != nil { + t.Fatalf("Execute after backup collision retry: %v", err) + } + if !collided || !strings.Contains(report.BackupPath, "000002") { + t.Fatalf("backup collision retry report=%+v collided=%v, want second suffix", report, collided) + } +} + +func TestRecoverBackupNameFailuresAreStructuredAndOperationLocal(t *testing.T) { + for _, tt := range []struct { + name string + configure func(*recoveryOperationDeps) + }{ + { + name: "random suffix failure", + configure: func(deps *recoveryOperationDeps) { + wantErr := errors.New("injected random suffix failure") + deps.randomHex = func(int) (string, error) { return "", wantErr } + }, + }, + { + name: "collision exhaustion", + configure: func(deps *recoveryOperationDeps) { + deps.randomHex = func(int) (string, error) { return "abcdef", nil } + deps.noClobberMove = func(string, string) error { return os.ErrExist } + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + dir := t.TempDir() + activePath := filepath.Join(dir, "active.db") + fromPath := filepath.Join(dir, "stranded.db") + createRecoveryDB(t, activePath, []storage.IndexedMessage{{Ordinal: 0, Role: "user", Text: "active backup naming", UUID: "12121212-1212-4212-8212-121212121212", ContentType: "text"}}) + createRecoveryDB(t, fromPath, []storage.IndexedMessage{{Ordinal: 0, Role: "assistant", Text: "stranded backup naming", UUID: "34343434-3434-4434-8434-343434343434", ContentType: "text"}}) + before := recoveryDirectoryInventory(t, dir) + deps := defaultRecoveryOperationDeps() + if tt.configure != nil { + tt.configure(&deps) + } + + _, err := executeWithDeps(context.Background(), Options{ActivePath: activePath, FromPath: fromPath}, deps) + if err == nil { + t.Fatal("executeWithDeps succeeded; want structured backup naming failure") + } + var failure *ApplyFailure + if !errors.As(err, &failure) { + t.Fatalf("error %T %[1]v, want *ApplyFailure", err) + } + if failure.Phase != phaseBackupMain || failure.State != replacementStateNoActiveMutation || !failure.NoActiveMutation { + t.Fatalf("failure phase/state/no-mutation = %s/%s/%v", failure.Phase, failure.State, failure.NoActiveMutation) + } + if after := recoveryDirectoryInventory(t, dir); !reflect.DeepEqual(after, before) { + t.Fatalf("naming failure mutated inventory\nbefore: %s\nafter: %s", describeRecoveryInventory(before), describeRecoveryInventory(after)) + } + }) + } +} + +func TestRecoverLeakedTempPathRemainsStructuredWhenCleanupFails(t *testing.T) { + dir := t.TempDir() + activePath := filepath.Join(dir, "active.db") + fromPath := filepath.Join(dir, "stranded.db") + createRecoveryDB(t, activePath, []storage.IndexedMessage{{Ordinal: 0, Role: "user", Text: "active leaked temp", UUID: "56565656-5656-4656-8656-565656565656", ContentType: "text"}}) + createRecoveryDB(t, fromPath, []storage.IndexedMessage{{Ordinal: 0, Role: "assistant", Text: "stranded leaked temp", UUID: "78787878-7878-4878-8878-787878787878", ContentType: "text"}}) + deps := defaultRecoveryOperationDeps() + deps.randomHex = deterministicRecoveryHex() + resolvedActive := canonicalRecoveryTestPath(t, activePath) + deps.installMove = func(oldPath, newPath string) error { + if strings.Contains(filepath.Base(oldPath), ".backscroll-recover-") && newPath == resolvedActive { + return fmt.Errorf("injected install failure before temp cleanup leak") + } + return noClobberMove(oldPath, newPath) + } + leakedTemp := "" + deps.cleanupFileSet = func(path string) error { + if strings.Contains(filepath.Base(path), ".backscroll-recover-") { + leakedTemp = path + return fmt.Errorf("injected cleanup failure for %s", path) + } + return removeSQLiteFileSet(path) + } + + _, err := executeWithDeps(context.Background(), Options{ActivePath: activePath, FromPath: fromPath}, deps) + if err == nil { + t.Fatal("executeWithDeps succeeded; want cleanup leak failure") + } + var failure *ApplyFailure + if !errors.As(err, &failure) { + t.Fatalf("error %T %[1]v, want *ApplyFailure", err) + } + if leakedTemp == "" || failure.TempPath != leakedTemp { + t.Fatalf("TempPath = %q, leakedTemp = %q", failure.TempPath, leakedTemp) + } + if failure.CleanupErr == nil || len(failure.CleanupErrors) == 0 { + t.Fatalf("CleanupErr/CleanupErrors = %v/%d, want structured cleanup failure", failure.CleanupErr, len(failure.CleanupErrors)) + } + if _, statErr := os.Lstat(leakedTemp); statErr != nil { + t.Fatalf("leaked temp %s missing after cleanup failure: %v", leakedTemp, statErr) + } +} + +func TestNoClobberMoveRejectsSymlinkCollision(t *testing.T) { + dir := t.TempDir() + source := filepath.Join(dir, "source.db") + target := filepath.Join(dir, "target.db") + elsewhere := filepath.Join(dir, "elsewhere.db") + if err := os.WriteFile(source, []byte("source"), 0o600); err != nil { + t.Fatalf("write source: %v", err) + } + if err := os.WriteFile(elsewhere, []byte("elsewhere"), 0o600); err != nil { + t.Fatalf("write elsewhere: %v", err) + } + if err := os.Symlink(elsewhere, target); err != nil { + t.Fatalf("symlink target: %v", err) + } + + err := noClobberMove(source, target) + if err == nil || !os.IsExist(err) { + t.Fatalf("noClobberMove error = %v, want existing-target failure", err) + } + if got, err := os.ReadFile(source); err != nil || string(got) != "source" { + t.Fatalf("source after collision = %q err=%v", got, err) + } + linkTarget, err := os.Readlink(target) + if err != nil || linkTarget != elsewhere { + t.Fatalf("target symlink = %q err=%v, want unchanged symlink", linkTarget, err) + } +} + +func TestRecoverCleanupErrorsAreSurfaced(t *testing.T) { + dir := t.TempDir() + activePath := filepath.Join(dir, "active.db") + fromPath := filepath.Join(dir, "stranded.db") + createRecoveryDB(t, activePath, []storage.IndexedMessage{{Ordinal: 0, Role: "user", Text: "active cleanup", UUID: "88888888-8888-4888-8888-888888888888", ContentType: "text"}}) + createRecoveryDB(t, fromPath, []storage.IndexedMessage{{Ordinal: 0, Role: "assistant", Text: "stranded cleanup", UUID: "99999999-9999-4999-8999-999999999999", ContentType: "text"}}) + badTempDir := filepath.Join(dir, ".backscroll-recover-bad.db") + if err := os.Mkdir(badTempDir, 0o700); err != nil { + t.Fatalf("mkdir bad temp dir: %v", err) + } + if err := os.WriteFile(filepath.Join(badTempDir, "child"), []byte("prevents cleanup"), 0o600); err != nil { + t.Fatalf("write bad temp child: %v", err) + } + deps := defaultRecoveryOperationDeps() + deps.createDestination = func(context.Context, string, compat.RecoveryPlan) (string, error) { return badTempDir, nil } + deps.syncFile = func(string) error { return fmt.Errorf("injected preinstall failure") } + + _, err := executeWithDeps(context.Background(), Options{ActivePath: activePath, FromPath: fromPath}, deps) + if err == nil || !strings.Contains(err.Error(), "cleanup verified recovery destination") { + t.Fatalf("Execute error = %v, want surfaced cleanup failure", err) + } +} + +func TestRecoverSameResolvedPathIsOneInput(t *testing.T) { + dir := t.TempDir() + activePath := filepath.Join(dir, "active.db") + createRecoveryDB(t, activePath, []storage.IndexedMessage{{ + Ordinal: 0, + Role: "user", + Text: "same database row", + UUID: "11111111-1111-4111-8111-111111111111", + Timestamp: "2026-08-18T00:00:00Z", + ContentType: "text", + }}) + + aliasPath := filepath.Join(dir, "alias.db") + if err := os.Symlink(activePath, aliasPath); err != nil { + t.Fatalf("symlink active database: %v", err) + } + + report, err := Execute(context.Background(), Options{ + ActivePath: activePath, + FromPath: aliasPath, + DryRun: true, + }) + if err != nil { + t.Fatalf("Execute dry run: %v", err) + } + + if !reflect.DeepEqual(report.InputCounts, []int{1}) { + t.Fatalf("InputCounts = %v, want one active input with one row", report.InputCounts) + } + if len(report.Shapes) != 1 { + t.Fatalf("Shapes = %d, want 1", len(report.Shapes)) + } + if report.FinalCount != 1 || report.ExactDuplicates != 0 { + t.Fatalf("FinalCount=%d ExactDuplicates=%d, want 1 and 0", report.FinalCount, report.ExactDuplicates) + } +} + +func TestExecuteWrapsImmutableLiveWALErrorWithRecoveryGuidance(t *testing.T) { + dir := t.TempDir() + activePath := filepath.Join(dir, "active.db") + active := createRecoveryLiveWALDB(t, activePath) + defer func() { _ = active.Close() }() + fromPath := filepath.Join(dir, "stranded.db") + createRecoveryDB(t, fromPath, []storage.IndexedMessage{{ + Ordinal: 0, + Role: "assistant", + Text: "stranded row", + UUID: "22222222-2222-4222-8222-222222222222", + Timestamp: "2026-08-18T00:01:00Z", + ContentType: "text", + }}) + + _, err := Execute(context.Background(), Options{ActivePath: activePath, FromPath: fromPath, DryRun: true}) + if err == nil { + t.Fatal("Execute with active live WAL succeeded; want immutable recovery planning guidance") + } + if !errors.Is(err, storage.ErrImmutableReadOnlyWALUnsafe) { + t.Fatalf("Execute error = %v, want ErrImmutableReadOnlyWALUnsafe", err) + } + message := err.Error() + resolvedActive, resolveErr := filepath.EvalSymlinks(activePath) + if resolveErr != nil { + t.Fatalf("resolve active path: %v", resolveErr) + } + for _, want := range []string{"recovery dry-run", "checkpoint", resolvedActive} { + if !strings.Contains(message, want) { + t.Fatalf("Execute error %q missing %q", message, want) + } + } +} + +func TestResolvePathSurfacesStatErrors(t *testing.T) { + dir := t.TempDir() + loop := filepath.Join(dir, "loop.db") + if err := os.Symlink(loop, loop); err != nil { + t.Fatalf("create symlink loop: %v", err) + } + + _, err := resolvePath(loop) + if err == nil { + t.Fatal("resolvePath symlink loop succeeded; want stat error") + } + if os.IsNotExist(err) { + t.Fatalf("resolvePath error = %v, want non-NotExist stat error", err) + } + if !strings.Contains(err.Error(), "stat database path") { + t.Fatalf("resolvePath error = %v, want stat database path context", err) + } +} + +func TestResolvePathAllowsMissingPathForOpenError(t *testing.T) { + missing := filepath.Join(t.TempDir(), "missing.db") + resolved, err := resolvePath(missing) + if err != nil { + t.Fatalf("resolvePath missing path: %v", err) + } + if resolved.path == "" || resolved.info != nil { + t.Fatalf("resolvePath missing = %+v, want path with nil info", resolved) + } +} + +type recoveryInventoryEntry struct { + Kind string + Size int64 + Mode os.FileMode + SHA256 string +} + +func deterministicRecoveryHex() func(int) (string, error) { + next := 1 + return func(int) (string, error) { + value := fmt.Sprintf("%06d", next) + next++ + return value, nil + } +} + +func recoveryDirectoryInventory(t *testing.T, dir string) map[string]recoveryInventoryEntry { + t.Helper() + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatalf("read dir inventory %s: %v", dir, err) + } + result := map[string]recoveryInventoryEntry{} + for _, entry := range entries { + path := filepath.Join(dir, entry.Name()) + info, err := entry.Info() + if err != nil { + t.Fatalf("stat inventory entry %s: %v", path, err) + } + item := recoveryInventoryEntry{Kind: "other", Size: info.Size(), Mode: info.Mode()} + if info.Mode().IsRegular() { + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read inventory entry %s: %v", path, err) + } + item.Kind = "regular" + item.SHA256 = fmt.Sprintf("%x", sha256.Sum256(data)) + } else if info.IsDir() { + item.Kind = "dir" + } else if info.Mode()&os.ModeSymlink != 0 { + item.Kind = "symlink" + } + result[entry.Name()] = item + } + return result +} + +func describeRecoveryInventory(inventory map[string]recoveryInventoryEntry) string { + keys := make([]string, 0, len(inventory)) + for key := range inventory { + keys = append(keys, key) + } + sort.Strings(keys) + parts := make([]string, 0, len(keys)) + for _, key := range keys { + entry := inventory[key] + parts = append(parts, fmt.Sprintf("%s:%s:%d:%s", key, entry.Kind, entry.Size, entry.SHA256)) + } + return strings.Join(parts, ",") +} + +func assertReplacementErrorPaths(t *testing.T, replacementErr *replacementError, activePath string) { + t.Helper() + if replacementErr.ActivePath != canonicalRecoveryTestPath(t, activePath) { + t.Fatalf("replacement active path = %q, want canonical active", replacementErr.ActivePath) + } + if replacementErr.TempPath == "" { + t.Fatalf("replacement temp path is empty: %+v", replacementErr) + } + if replacementErr.BackupPath == "" && replacementErr.BackupVisible { + t.Fatalf("backup is visible without exact backup path: %+v", replacementErr) + } +} + +func assertNoRecoveryTemps(t *testing.T, dir string) { + t.Helper() + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatalf("read dir for temp assertion: %v", err) + } + for _, entry := range entries { + if strings.Contains(entry.Name(), ".backscroll-recover-") { + t.Fatalf("temporary recovery destination still visible: %s in %s", entry.Name(), describeRecoveryInventory(recoveryDirectoryInventory(t, dir))) + } + } +} + +func assertRecoveryFaultInventory(t *testing.T, want, dir, activePath, fromPath, backupPath string, activeBefore recoveryDBFileSnapshot, before map[string]recoveryInventoryEntry, reportedBackup string) { + t.Helper() + after := recoveryDirectoryInventory(t, dir) + switch want { + case "unchanged": + if !reflect.DeepEqual(after, before) { + t.Fatalf("failure inventory changed\nbefore: %s\nafter: %s", describeRecoveryInventory(before), describeRecoveryInventory(after)) + } + case "backup-only": + if backupPath == "" { + t.Fatal("backup-only inventory requested without backup path") + } + wantInventory := cloneRecoveryInventory(before) + delete(wantInventory, filepath.Base(activePath)) + wantInventory[filepath.Base(backupPath)] = recoveryInventoryEntry{Kind: "regular", Size: int64(len(activeBefore.main.data)), Mode: activeBefore.main.mode, SHA256: strings.TrimPrefix(activeBefore.main.fingerprint, "sha256:")} + if !reflect.DeepEqual(after, wantInventory) { + t.Fatalf("backup-only inventory mismatch\nwant: %s\ngot: %s", describeRecoveryInventory(wantInventory), describeRecoveryInventory(after)) + } + case "replacement-and-backup": + if backupPath == "" || reportedBackup != backupPath { + t.Fatalf("replacement failure backup path = err %q report %q, want same non-empty", backupPath, reportedBackup) + } + if _, err := os.Stat(backupPath); err != nil { + t.Fatalf("stat visible backup %s: %v", backupPath, err) + } + if _, err := os.Stat(activePath); err != nil { + t.Fatalf("stat visible replacement active %s: %v", activePath, err) + } + if _, ok := after[filepath.Base(backupPath)]; !ok { + t.Fatalf("replacement inventory lacks backup %s: %s", filepath.Base(backupPath), describeRecoveryInventory(after)) + } + if _, ok := after[filepath.Base(activePath)]; !ok { + t.Fatalf("replacement inventory lacks active: %s", describeRecoveryInventory(after)) + } + if _, ok := after[filepath.Base(fromPath)]; !ok { + t.Fatalf("replacement inventory lacks stranded source: %s", describeRecoveryInventory(after)) + } + if len(after) != 3 { + t.Fatalf("replacement inventory has extra entries: %s", describeRecoveryInventory(after)) + } + default: + t.Fatalf("unknown inventory expectation %q", want) + } +} + +func cloneRecoveryInventory(in map[string]recoveryInventoryEntry) map[string]recoveryInventoryEntry { + out := make(map[string]recoveryInventoryEntry, len(in)) + for key, value := range in { + out[key] = value + } + return out +} + +func appendRecoveryTestBytes(t *testing.T, path, text string) { + t.Helper() + f, err := os.OpenFile(path, os.O_WRONLY|os.O_APPEND, 0) + if err != nil { + t.Fatalf("open %s for append mutation: %v", path, err) + } + if _, err := f.WriteString(text); err != nil { + _ = f.Close() + t.Fatalf("append mutation to %s: %v", path, err) + } + if err := f.Close(); err != nil { + t.Fatalf("close appended mutation %s: %v", path, err) + } +} + +func replaceRecoveryTestFile(t *testing.T, path string, data []byte) { + t.Helper() + temp := path + ".swap-test" + if err := os.WriteFile(temp, data, 0o600); err != nil { + t.Fatalf("write swap file %s: %v", temp, err) + } + if err := os.Rename(temp, path); err != nil { + t.Fatalf("rename swap file over %s: %v", path, err) + } +} + +func createRecoveryDB(t *testing.T, path string, messages []storage.IndexedMessage) { + t.Helper() + db := openRecoveryDB(t, path, messages) + if err := db.Close(); err != nil { + t.Fatalf("close test database %s: %v", path, err) + } +} + +func createRecoveryLiveWALDB(t *testing.T, path string) *storage.Database { + t.Helper() + db := openRecoveryDB(t, path, []storage.IndexedMessage{{ + Ordinal: 0, + Role: "user", + Text: "active live WAL row", + UUID: "11111111-1111-4111-8111-111111111111", + Timestamp: "2026-08-18T00:00:00Z", + ContentType: "text", + }}) + wal, err := os.Stat(path + "-wal") + if err != nil { + _ = db.Close() + t.Fatalf("stat live WAL sidecar: %v", err) + } + if wal.Size() == 0 { + _ = db.Close() + t.Fatal("live WAL sidecar is empty; test fixture did not create committed WAL frames") + } + return db +} + +type recoveryDBFileSnapshot struct { + main recoveryFileSnapshot + sidecars map[string]recoveryFileSnapshot +} + +type recoveryFileSnapshot struct { + data []byte + mode os.FileMode + mtime time.Time + fingerprint string +} + +func snapshotRecoveryDBFile(t *testing.T, dbPath string) recoveryDBFileSnapshot { + t.Helper() + return recoveryDBFileSnapshot{ + main: snapshotRecoveryFile(t, dbPath), + sidecars: snapshotRecoverySidecars(t, dbPath), + } +} + +func snapshotRecoverySidecars(t *testing.T, dbPath string) map[string]recoveryFileSnapshot { + t.Helper() + result := map[string]recoveryFileSnapshot{} + for _, suffix := range []string{"-wal", "-shm"} { + path := dbPath + suffix + if _, err := os.Stat(path); os.IsNotExist(err) { + continue + } + result[suffix] = snapshotRecoveryFile(t, path) + } + return result +} + +func snapshotRecoveryFile(t *testing.T, path string) recoveryFileSnapshot { + t.Helper() + info, err := os.Stat(path) + if err != nil { + t.Fatalf("stat %s: %v", path, err) + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + return recoveryFileSnapshot{data: data, mode: info.Mode(), mtime: info.ModTime(), fingerprint: fmt.Sprintf("sha256:%x", sha256.Sum256(data))} +} + +func assertRecoveryDBFileSnapshot(t *testing.T, label, dbPath string, want recoveryDBFileSnapshot) { + t.Helper() + got := snapshotRecoveryDBFile(t, dbPath) + if !reflect.DeepEqual(got, want) { + t.Fatalf("%s changed\nwant: %s\ngot: %s", label, describeRecoveryDBFileSnapshot(want), describeRecoveryDBFileSnapshot(got)) + } +} + +func describeRecoveryDBFileSnapshot(snapshot recoveryDBFileSnapshot) string { + return fmt.Sprintf("main=%s sidecars=%v", describeRecoveryFileSnapshot(snapshot.main), recoverySnapshotSidecarKeys(snapshot.sidecars)) +} + +func describeRecoveryFileSnapshot(snapshot recoveryFileSnapshot) string { + return fmt.Sprintf("bytes=%d mode=%s mtime=%s fingerprint=%s", len(snapshot.data), snapshot.mode, snapshot.mtime.Format(time.RFC3339Nano), snapshot.fingerprint) +} + +func recoverySnapshotSidecarKeys(snapshot map[string]recoveryFileSnapshot) []string { + keys := make([]string, 0, len(snapshot)) + for key := range snapshot { + keys = append(keys, key) + } + return keys +} + +func canonicalRecoveryTestPath(t *testing.T, path string) string { + t.Helper() + abs, err := filepath.Abs(path) + if err != nil { + t.Fatalf("absolute path for %s: %v", path, err) + } + resolved, err := filepath.EvalSymlinks(abs) + if err == nil { + return resolved + } + parent, parentErr := filepath.EvalSymlinks(filepath.Dir(abs)) + if parentErr != nil { + t.Fatalf("resolve path for %s: %v", abs, err) + } + return filepath.Join(parent, filepath.Base(abs)) +} + +func assertRecoveryTexts(t *testing.T, dbPath string, want []string) { + t.Helper() + db, err := storage.OpenReadOnly(dbPath) + if err != nil { + t.Fatalf("open recovered database read-only: %v", err) + } + defer func() { _ = db.Close() }() + rows, err := db.DB().Query(`SELECT text FROM search_items ORDER BY text`) + if err != nil { + t.Fatalf("query recovered texts: %v", err) + } + defer func() { _ = rows.Close() }() + var got []string + for rows.Next() { + var text string + if err := rows.Scan(&text); err != nil { + t.Fatalf("scan recovered text: %v", err) + } + got = append(got, text) + } + if err := rows.Err(); err != nil { + t.Fatalf("read recovered texts: %v", err) + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("recovered texts = %v, want %v", got, want) + } +} + +func openRecoveryDB(t *testing.T, path string, messages []storage.IndexedMessage) *storage.Database { + t.Helper() + db, err := storage.Open(path) + if err != nil { + t.Fatalf("open test database %s: %v", path, err) + } + if err := db.SyncFiles([]storage.IndexedFile{{ + SourcePath: "/sessions/shared.jsonl", + Source: "session", + Hash: "hash-" + filepath.Base(path), + Project: "project", + Messages: messages, + }}); err != nil { + _ = db.Close() + t.Fatalf("sync test database %s: %v", path, err) + } + return db +} diff --git a/internal/sequences/prefixspan.go b/internal/sequences/prefixspan.go index 01c702a..e527209 100644 --- a/internal/sequences/prefixspan.go +++ b/internal/sequences/prefixspan.go @@ -58,7 +58,7 @@ func Mine(sequences []Sequence, minSupport, minLen, maxLen int) []Pattern { // Recursively extend each 1-pattern (stop if maxLen reached) for _, base := range onePatterns { if len(base.Items) < maxLen { - extended := mineExtensions(sequences, base, minSupport, minLen, maxLen) + extended := mineExtensions(projectAll(sequences, base.Items), base, minSupport, minLen, maxLen) patterns = append(patterns, extended...) } } @@ -76,15 +76,14 @@ func Mine(sequences []Sequence, minSupport, minLen, maxLen int) []Pattern { // mineExtensions finds longer patterns by extending a base pattern. // Stops recursing when len(extended) >= maxLen to prevent combinatorial explosion. -func mineExtensions(sequences []Sequence, base Pattern, minSupport, minLen, maxLen int) []Pattern { - // Compute projected database: sessions containing base, positioned after base's last item - var projected []Sequence - for _, seq := range sequences { - if proj := project(seq, base.Items); len(proj) > 0 { - projected = append(projected, Sequence{SessionID: seq.SessionID, Items: proj}) - } - } - +// +// projected MUST already be the database positioned after base — that is the whole +// point of PrefixSpan. Recursing on the full database instead re-scans and re-projects +// every session at every depth, which is why `--all-projects` used to hang: 305 +// sessions and 36,956 events were re-projected once per node of the search tree. +// Incremental projection is equivalent because project() completes a pattern greedily +// left-to-right, so projecting by [a,b] equals projecting by [a] and then by [b]. +func mineExtensions(projected []Sequence, base Pattern, minSupport, minLen, maxLen int) []Pattern { if len(projected) == 0 { return nil } @@ -116,9 +115,11 @@ func mineExtensions(sequences []Sequence, base Pattern, minSupport, minLen, maxL }) } - // Recursively extend further (stop if maxLen reached) + // Recursively extend further (stop if maxLen reached), projecting the + // CURRENT database by the single new item rather than re-deriving the + // whole prefix from the original sequences. if len(extended) < maxLen { - subExtended := mineExtensions(sequences, Pattern{Items: extended}, minSupport, minLen, maxLen) + subExtended := mineExtensions(projectAll(projected, []string{item}), Pattern{Items: extended}, minSupport, minLen, maxLen) patterns = append(patterns, subExtended...) } } @@ -167,3 +168,15 @@ func lexicographic(items []string) string { } return string(buf) } + +// projectAll projects every sequence in db by pattern, dropping the ones where the +// pattern does not complete or leaves no suffix. +func projectAll(db []Sequence, pattern []string) []Sequence { + var out []Sequence + for _, seq := range db { + if proj := project(seq, pattern); len(proj) > 0 { + out = append(out, Sequence{SessionID: seq.SessionID, Items: proj}) + } + } + return out +} diff --git a/internal/sequences/prefixspan_equivalence_test.go b/internal/sequences/prefixspan_equivalence_test.go new file mode 100644 index 0000000..58803d9 --- /dev/null +++ b/internal/sequences/prefixspan_equivalence_test.go @@ -0,0 +1,114 @@ +package sequences + +import ( + "fmt" + "math/rand" + "sort" + "testing" +) + +// TestEquivalenceVsFullDBRecursion pins that projecting incrementally produces exactly +// the patterns the original full-database recursion did. mineReference below is that +// original algorithm, kept verbatim as the specification of correctness: the fix is a +// performance change and any divergence in output is a bug, not an improvement. +func TestEquivalenceVsFullDBRecursion(t *testing.T) { + rng := rand.New(rand.NewSource(42)) + items := []string{"GIT", "TEST_EXEC", "FILE_READ", "FILE_WRITE", "SEARCH", "NAV", "GO_EXEC"} + for trial := 0; trial < 200; trial++ { + n := 3 + rng.Intn(12) + var seqs []Sequence + for i := 0; i < n; i++ { + l := 1 + rng.Intn(14) + var it []string + for j := 0; j < l; j++ { + it = append(it, items[rng.Intn(len(items))]) + } + seqs = append(seqs, Sequence{SessionID: fmt.Sprintf("s%d", i), Items: it}) + } + ms := 1 + rng.Intn(3) + got := Mine(seqs, ms, 1, 4) + want := mineReference(seqs, ms, 1, 4) + if key(got) != key(want) { + t.Fatalf("trial %d diverged\n got: %s\nwant: %s", trial, key(got), key(want)) + } + } +} + +func key(ps []Pattern) string { + var out []string + for _, p := range ps { + out = append(out, fmt.Sprintf("%v:%d", p.Items, p.Support)) + } + sort.Strings(out) + return fmt.Sprint(out) +} + +// mineReference is the pre-fix algorithm: recursion over the FULL database. +func mineReference(sequences []Sequence, minSupport, minLen, maxLen int) []Pattern { + if minSupport < 1 || minLen < 1 || maxLen < 1 || len(sequences) == 0 || + minSupport > len(sequences) || minLen > maxLen { + return nil + } + var patterns []Pattern + freq := map[string]int{} + for _, seq := range sequences { + seen := map[string]bool{} + for _, it := range seq.Items { + if !seen[it] { + freq[it]++ + seen[it] = true + } + } + } + var one []Pattern + for it, c := range freq { + if c >= minSupport { + one = append(one, Pattern{Items: []string{it}, Support: c}) + } + } + if minLen == 1 { + patterns = append(patterns, one...) + } + for _, b := range one { + if len(b.Items) < maxLen { + patterns = append(patterns, refExt(sequences, b, minSupport, minLen, maxLen)...) + } + } + return patterns +} + +func refExt(sequences []Sequence, base Pattern, minSupport, minLen, maxLen int) []Pattern { + var projected []Sequence + for _, seq := range sequences { + if pr := project(seq, base.Items); len(pr) > 0 { + projected = append(projected, Sequence{SessionID: seq.SessionID, Items: pr}) + } + } + if len(projected) == 0 { + return nil + } + freq := map[string]int{} + for _, seq := range projected { + seen := map[string]bool{} + for _, it := range seq.Items { + if !seen[it] { + freq[it]++ + seen[it] = true + } + } + } + var patterns []Pattern + for it, c := range freq { + if c >= minSupport { + ext := append([]string(nil), base.Items...) + ext = append(ext, it) + if len(ext) >= minLen { + patterns = append(patterns, Pattern{Items: ext, Support: c}) + } + if len(ext) < maxLen { + patterns = append(patterns, refExt(sequences, Pattern{Items: ext}, minSupport, minLen, maxLen)...) + } + } + } + return patterns +} diff --git a/internal/storage/migration_plan.go b/internal/storage/migration_plan.go new file mode 100644 index 0000000..e7dea89 --- /dev/null +++ b/internal/storage/migration_plan.go @@ -0,0 +1,557 @@ +package storage + +import ( + "context" + "crypto/sha256" + "database/sql" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/pablontiv/backscroll/internal/compat" +) + +var ( + snapshotDatabase = SnapshotDatabase + beginMigrationTx = func(ctx context.Context, db *sql.DB) (*sql.Tx, error) { + return db.BeginTx(ctx, nil) + } +) + +// SnapshotDatabase creates a read-only reopenable sibling snapshot of srcPath. +func SnapshotDatabase(ctx context.Context, srcPath string) (string, error) { + targetPath, err := nextSnapshotPath(srcPath) + if err != nil { + return "", err + } + + db, err := sql.Open("sqlite", "file:"+srcPath+"?mode=ro&_pragma=busy_timeout(5000)") + if err != nil { + return "", fmt.Errorf("open snapshot source: %w", err) + } + defer db.Close() + + if _, err := db.ExecContext(ctx, "VACUUM INTO "+quoteSQLString(targetPath)); err != nil { + _ = os.Remove(targetPath) + return "", fmt.Errorf("create database snapshot: %w", err) + } + if err := fsyncPath(targetPath); err != nil { + return "", err + } + if err := fsyncPath(filepath.Dir(targetPath)); err != nil { + return "", err + } + return targetPath, nil +} + +// ApplyMigrationPlan applies a checked compatibility migration plan atomically. +func (d *Database) ApplyMigrationPlan(ctx context.Context, plan compat.MigrationPlan) error { + if len(plan.Steps) == 0 { + return nil + } + if d.path == "" { + return fmt.Errorf("database path is not bound to receiver") + } + + if planIncludesVersion(plan, 9) { + if err := prepareV9ToolEventDuplicates(ctx, d.db); err != nil { + return err + } + } + + if planHasDestructiveMigration(plan) { + if _, err := snapshotDatabase(ctx, d.path); err != nil { + return err + } + } + + v6Recorded := plan.From.AppliedVersion >= 6 || planIncludesVersion(plan, 6) + tx, err := beginMigrationTx(ctx, d.db) + if err != nil { + return fmt.Errorf("begin migration plan transaction: %w", err) + } + defer func() { _ = tx.Rollback() }() + + livePlan, diag, err := compat.InspectIndex(ctx, tx) + if err != nil { + return fmt.Errorf("re-inspect schema in migration transaction: %w", err) + } + if livePlan.From.Signature != plan.From.Signature { + return fmt.Errorf("index schema changed since inspection: got %s want %s", livePlan.From.Signature, plan.From.Signature) + } + if diag != nil && !planStartsFromEmptySchema(plan) { + return fmt.Errorf("re-inspect schema in migration transaction: %s: %s", diag.Code, diag.Summary) + } + if planIncludesVersion(plan, 9) { + if err := prepareV9ToolEventDuplicates(ctx, tx); err != nil { + return err + } + } + + for _, step := range plan.Steps { + if step.Version > 6 && !v6Recorded { + if err := recordMigration(ctx, tx, 6, "V6 drop phantom source_metadata column", sqlV6Drop, "record migration v6"); err != nil { + return err + } + v6Recorded = true + } + + apply, ok := migrationPlanDispatch[step] + if !ok { + return fmt.Errorf("unsupported migration step %d %q", step.Version, step.Name) + } + if err := apply(ctx, tx, plan.From); err != nil { + return err + } + } + + if err := compat.VerifyCurrentShape(ctx, tx); err != nil { + return fmt.Errorf("verify final schema before commit: %w", err) + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("commit migration plan: %w", err) + } + + verify, err := OpenReadOnly(d.path) + if err != nil { + return fmt.Errorf("reopen migrated database read-only: %w", err) + } + defer func() { _ = verify.Close() }() + if err := compat.VerifyCurrentShape(ctx, verify.DB()); err != nil { + return fmt.Errorf("verify committed schema: %w", err) + } + return nil +} + +type migrationApplier func(context.Context, *sql.Tx, compat.SchemaShape) error + +func (d *Database) applySingleMigration(apply migrationApplier) error { + tx, err := d.db.Begin() + if err != nil { + return fmt.Errorf("begin transaction: %w", err) + } + defer func() { _ = tx.Rollback() }() + if err := apply(context.Background(), tx, compat.SchemaShape{}); err != nil { + return err + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("commit migration: %w", err) + } + return nil +} + +var migrationPlanDispatch = map[compat.MigrationStep]migrationApplier{ + {Version: 1, Name: "V1 core schema"}: applyV1, + {Version: 2, Name: "V2 embedding tables"}: applyV2, + {Version: 3, Name: "V3 embedding blob column"}: applyV3, + {Version: 4, Name: "V4 tool_fts trigram index"}: applyV4, + {Version: 5, Name: "V5 drop phantom session_events"}: applyV5, + {Version: 6, Name: "V6 drop source_metadata when present"}: applyV6, + {Version: 7, Name: "V7 reasoning triggers"}: applyV7, + {Version: 8, Name: "V8 perennity: extraction_version, was_interrupted, tool_events"}: applyV8, + {Version: 9, Name: "V9 tool_events uuid uniqueness index"}: applyV9, + {Version: 10, Name: "V10 template mining: message_templates, template_matches"}: applyV10, + {Version: 11, Name: "V11 correction detection: correction_signals"}: applyV11, + {Version: 12, Name: "V12 agent classification: annotations"}: applyV12, + {Version: 13, Name: "V13 backfill discovery indexes"}: applyV13, +} + +func isDestructiveMigration(step compat.MigrationStep) bool { + return step.Version == 5 || step.Version == 6 || step.Version == 8 || step.Version == 9 +} + +func planHasDestructiveMigration(plan compat.MigrationPlan) bool { + for _, step := range plan.Steps { + if isDestructiveMigration(step) { + return true + } + } + return false +} + +func planStartsFromEmptySchema(plan compat.MigrationPlan) bool { + return plan.From.AppliedVersion == 0 && len(plan.Steps) > 0 && plan.Steps[0].Version == 1 +} + +func planIncludesVersion(plan compat.MigrationPlan, version int) bool { + for _, step := range plan.Steps { + if step.Version == version { + return true + } + } + return false +} + +func nextSnapshotPath(srcPath string) (string, error) { + for i := 0; ; i++ { + candidate := srcPath + ".snapshot" + if i > 0 { + candidate = fmt.Sprintf("%s.snapshot.%d", srcPath, i) + } + _, err := os.Stat(candidate) + if os.IsNotExist(err) { + return candidate, nil + } + if err != nil { + return "", fmt.Errorf("stat snapshot target: %w", err) + } + } +} + +func quoteSQLString(value string) string { + return "'" + strings.ReplaceAll(value, "'", "''") + "'" +} + +func fsyncPath(path string) error { + file, err := os.Open(path) + if err != nil { + return fmt.Errorf("open for fsync %s: %w", path, err) + } + defer file.Close() + if err := file.Sync(); err != nil { + return fmt.Errorf("fsync %s: %w", path, err) + } + return nil +} + +func applyV1(ctx context.Context, tx *sql.Tx, from compat.SchemaShape) error { + _ = from + if _, err := tx.ExecContext(ctx, sqlV1Core); err != nil { + return fmt.Errorf("create core tables: %w", err) + } + if _, err := tx.ExecContext(ctx, sqlV1FTS5); err != nil { + return fmt.Errorf("create FTS5 virtual table: %w", err) + } + if _, err := tx.ExecContext(ctx, sqlV1Triggers); err != nil { + return fmt.Errorf("create triggers: %w", err) + } + return recordMigration(ctx, tx, 1, "V1 core schema", sqlV1CoreDDL, "record migration") +} + +func applyV2(ctx context.Context, tx *sql.Tx, from compat.SchemaShape) error { + _ = from + if _, err := tx.ExecContext(ctx, sqlV2); err != nil { + return fmt.Errorf("create embedding tables: %w", err) + } + return recordMigration(ctx, tx, 2, "V2 embedding tables", sqlV2, "record migration v2") +} + +func applyV3(ctx context.Context, tx *sql.Tx, from compat.SchemaShape) error { + _ = from + if _, err := tx.ExecContext(ctx, sqlV3); err != nil { + return fmt.Errorf("add embedding column: %w", err) + } + return recordMigration(ctx, tx, 3, "V3 embedding blob column", sqlV3, "record migration v3") +} + +func applyV4(ctx context.Context, tx *sql.Tx, from compat.SchemaShape) error { + _ = from + if _, err := tx.ExecContext(ctx, sqlV4ToolFTS); err != nil { + return fmt.Errorf("create tool_fts: %w", err) + } + if _, err := tx.ExecContext(ctx, sqlV4Triggers); err != nil { + return fmt.Errorf("rebuild triggers: %w", err) + } + if _, err := tx.ExecContext(ctx, sqlV4Repopulate); err != nil { + return fmt.Errorf("repopulate indexes: %w", err) + } + return recordMigration(ctx, tx, 4, "V4 tool_fts trigram index", sqlV4ToolFTS+sqlV4Triggers, "record migration v4") +} + +func applyV5(ctx context.Context, tx *sql.Tx, from compat.SchemaShape) error { + _ = from + if _, err := tx.ExecContext(ctx, sqlV5Drop); err != nil { + return fmt.Errorf("drop session_events: %w", err) + } + return recordMigration(ctx, tx, 5, "V5 drop phantom session_events", sqlV5Drop, "record migration v5") +} + +func applyV6(ctx context.Context, tx *sql.Tx, from compat.SchemaShape) error { + _ = from + if _, err := tx.ExecContext(ctx, sqlV6Drop); err != nil { + return fmt.Errorf("drop source_metadata column: %w", err) + } + return recordMigration(ctx, tx, 6, "V6 drop phantom source_metadata column", sqlV6Drop, "record migration v6") +} + +func applyV7(ctx context.Context, tx *sql.Tx, from compat.SchemaShape) error { + _ = from + if _, err := tx.ExecContext(ctx, sqlV7Triggers); err != nil { + return fmt.Errorf("rebuild triggers for reasoning: %w", err) + } + return recordMigration(ctx, tx, 7, "V7 reasoning content_type routes to messages_fts", sqlV7Triggers, "record migration v7") +} + +func applyV8(ctx context.Context, tx *sql.Tx, from compat.SchemaShape) error { + _ = from + if _, err := tx.ExecContext(ctx, sqlV8); err != nil { + return fmt.Errorf("apply v8 perennity schema: %w", err) + } + if _, err := tx.ExecContext(ctx, sqlV8SearchItemsRebuild); err != nil { + return fmt.Errorf("rebuild search_items for v8 shape: %w", err) + } + if _, err := tx.ExecContext(ctx, sqlV7Triggers); err != nil { + return fmt.Errorf("restore search_items triggers after v8 rebuild: %w", err) + } + if _, err := tx.ExecContext(ctx, sqlV4Repopulate); err != nil { + return fmt.Errorf("repopulate indexes after v8 rebuild: %w", err) + } + return recordMigration(ctx, tx, 8, "V8 perennity: extraction_version, was_interrupted, tool_events", sqlV8, "record migration v8") +} + +func applyV9(ctx context.Context, tx *sql.Tx, from compat.SchemaShape) error { + _ = from + if _, err := tx.ExecContext(ctx, sqlV9); err != nil { + return fmt.Errorf("apply v9 tool_events uuid uniqueness: %w", err) + } + return recordMigration(ctx, tx, 9, "V9 tool_events uuid uniqueness index", sqlV9, "record migration v9") +} + +type queryContexter interface { + QueryContext(context.Context, string, ...any) (*sql.Rows, error) +} + +func prepareV9ToolEventDuplicates(ctx context.Context, q queryContexter) error { + existsRows, err := q.QueryContext(ctx, `SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'tool_events' LIMIT 1`) + if err != nil { + return fmt.Errorf("inspect V9 tool_events table: %w", err) + } + toolEventsExists := existsRows.Next() + if err := existsRows.Err(); err != nil { + _ = existsRows.Close() + return fmt.Errorf("read V9 tool_events table: %w", err) + } + if err := existsRows.Close(); err != nil { + return fmt.Errorf("close V9 tool_events table probe: %w", err) + } + if !toolEventsExists { + return nil + } + + rows, err := q.QueryContext(ctx, ` + SELECT + message_uuid, + source_path, + ordinal, + tool_name, + command_head, + is_error, + exit_code, + extraction_version + FROM tool_events + WHERE message_uuid IS NOT NULL + ORDER BY message_uuid, id + `) + if err != nil { + return fmt.Errorf("inspect V9 tool_events duplicates: %w", err) + } + defer rows.Close() + + seen := map[string]v9ToolEventRow{} + for rows.Next() { + var payload v9ToolEventRow + if err := rows.Scan( + &payload.MessageUUID, + &payload.SourcePath, + &payload.Ordinal, + &payload.ToolName, + &payload.CommandHead, + &payload.IsError, + &payload.ExitCode, + &payload.ExtractionVersion, + ); err != nil { + return fmt.Errorf("scan V9 tool_events duplicates: %w", err) + } + prior, ok := seen[payload.MessageUUID] + if !ok { + seen[payload.MessageUUID] = payload + continue + } + if prior != payload { + return fmt.Errorf("conflicting tool_events for message_uuid %q before V9 uniqueness migration", payload.MessageUUID) + } + } + if err := rows.Err(); err != nil { + return fmt.Errorf("read V9 tool_events duplicates: %w", err) + } + return nil +} + +type v9ToolEventRow struct { + MessageUUID string + SourcePath string + Ordinal int64 + ToolName string + CommandHead sql.NullString + IsError sql.NullInt64 + ExitCode sql.NullInt64 + ExtractionVersion int64 +} + +func applyV10(ctx context.Context, tx *sql.Tx, from compat.SchemaShape) error { + _ = from + if _, err := tx.ExecContext(ctx, sqlV10); err != nil { + return fmt.Errorf("apply v10 template-mining schema: %w", err) + } + return recordMigration(ctx, tx, 10, "V10 template mining: message_templates, template_matches", sqlV10, "record migration v10") +} + +func applyV11(ctx context.Context, tx *sql.Tx, from compat.SchemaShape) error { + _ = from + if _, err := tx.ExecContext(ctx, sqlV11); err != nil { + return fmt.Errorf("apply v11 correction_signals schema: %w", err) + } + return recordMigration(ctx, tx, 11, "V11 correction detection: correction_signals", sqlV11, "record migration v11") +} + +func applyV12(ctx context.Context, tx *sql.Tx, from compat.SchemaShape) error { + _ = from + if _, err := tx.ExecContext(ctx, sqlV12); err != nil { + return fmt.Errorf("apply v12 annotations schema: %w", err) + } + return recordMigration(ctx, tx, 12, "V12 agent classification: annotations (free-form labels; enum freeze deferred)", sqlV12, "record migration v12") +} + +func applyV13(ctx context.Context, tx *sql.Tx, from compat.SchemaShape) error { + _ = from + if _, err := tx.ExecContext(ctx, sqlV13); err != nil { + return fmt.Errorf("apply v13 indexes: %w", err) + } + return recordMigration(ctx, tx, 13, "V13 backfill discovery indexes", sqlV13, "record migration v13") +} + +func recordMigration(ctx context.Context, tx *sql.Tx, version int, name string, body string, errorPrefix string) error { + checksum := sha256.Sum256([]byte(body)) + checksumHex := fmt.Sprintf("%x", checksum) + if _, err := tx.ExecContext(ctx, ` + INSERT INTO schema_migrations (version, name, applied_on, checksum) + VALUES (?, ?, CURRENT_TIMESTAMP, ?) + `, version, name, checksumHex); err != nil { + return fmt.Errorf("%s: %w", errorPrefix, err) + } + return nil +} + +const sqlV5Drop = ` +DROP INDEX IF EXISTS idx_session_events_order; +DROP INDEX IF EXISTS idx_session_events_project; +DROP TABLE IF EXISTS session_events; +` + +const sqlV6Drop = `ALTER TABLE search_items DROP COLUMN source_metadata;` + +const sqlV8 = ` +ALTER TABLE search_items ADD COLUMN extraction_version INTEGER; +ALTER TABLE search_items ADD COLUMN was_interrupted INTEGER; +CREATE TABLE IF NOT EXISTS tool_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + message_uuid TEXT, + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + tool_name TEXT NOT NULL, + command_head TEXT, + is_error INTEGER, + exit_code INTEGER, + extraction_version INTEGER NOT NULL, + UNIQUE(source_path, ordinal) +); +CREATE INDEX IF NOT EXISTS idx_tool_events_tool ON tool_events(tool_name); +CREATE INDEX IF NOT EXISTS idx_tool_events_uuid ON tool_events(message_uuid); +` + +const sqlV8SearchItemsRebuild = ` +DROP TRIGGER IF EXISTS search_items_ai; +DROP TRIGGER IF EXISTS search_items_ad; +DROP TRIGGER IF EXISTS search_items_au; +ALTER TABLE search_items RENAME TO search_items_v8_old; +CREATE TABLE search_items ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source TEXT NOT NULL DEFAULT 'session', + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + role TEXT NOT NULL, + text TEXT NOT NULL, + timestamp TEXT, + uuid TEXT UNIQUE, + project TEXT, + content_type TEXT NOT NULL DEFAULT 'text', + extraction_version INTEGER, + was_interrupted INTEGER +); +INSERT INTO search_items (id, source, source_path, ordinal, role, text, timestamp, uuid, project, content_type, extraction_version, was_interrupted) + SELECT id, source, source_path, ordinal, role, text, timestamp, uuid, project, content_type, extraction_version, was_interrupted + FROM search_items_v8_old; +DROP TABLE search_items_v8_old; +CREATE INDEX IF NOT EXISTS idx_search_items_source_path ON search_items(source_path); +CREATE INDEX IF NOT EXISTS idx_search_items_project ON search_items(project); +` + +const sqlV9 = ` +DELETE FROM tool_events WHERE message_uuid IS NOT NULL AND id NOT IN ( + SELECT MIN(id) FROM tool_events WHERE message_uuid IS NOT NULL GROUP BY message_uuid +); +CREATE UNIQUE INDEX IF NOT EXISTS idx_tool_events_uuid_unique ON tool_events(message_uuid) WHERE message_uuid IS NOT NULL; +` + +const sqlV10 = ` +CREATE TABLE IF NOT EXISTS message_templates ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + signature TEXT UNIQUE NOT NULL, + normalization_version INTEGER NOT NULL, + template_text TEXT NOT NULL, + occurrence_count INTEGER NOT NULL DEFAULT 1, + first_seen TEXT, + last_seen TEXT +); +CREATE INDEX IF NOT EXISTS idx_templates_sig ON message_templates(signature); +CREATE INDEX IF NOT EXISTS idx_templates_version ON message_templates(normalization_version); + +CREATE TABLE IF NOT EXISTS template_matches ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + template_id INTEGER NOT NULL, + item_uuid TEXT, + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + UNIQUE(source_path, ordinal, template_id), + FOREIGN KEY(template_id) REFERENCES message_templates(id) +); +CREATE INDEX IF NOT EXISTS idx_matches_template ON template_matches(template_id); +CREATE INDEX IF NOT EXISTS idx_matches_uuid ON template_matches(item_uuid); +` + +const sqlV11 = ` +CREATE TABLE IF NOT EXISTS correction_signals ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + item_uuid TEXT, + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + detector TEXT NOT NULL, + confidence REAL NOT NULL, + extraction_version INTEGER NOT NULL, + UNIQUE(source_path, ordinal, detector) +); +CREATE INDEX IF NOT EXISTS idx_correction_signals_detector ON correction_signals(detector); +CREATE INDEX IF NOT EXISTS idx_correction_signals_confidence ON correction_signals(confidence DESC); +` + +const sqlV12 = ` +CREATE TABLE IF NOT EXISTS annotations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + item_uuid TEXT, + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + kind TEXT NOT NULL, + label TEXT NOT NULL, + source TEXT NOT NULL DEFAULT 'agent', + created_at TEXT NOT NULL, + UNIQUE(source_path, ordinal, kind) +); +CREATE INDEX IF NOT EXISTS idx_annotations_uuid ON annotations(item_uuid); +CREATE INDEX IF NOT EXISTS idx_annotations_kind ON annotations(kind); +` + +const sqlV13 = ` +CREATE INDEX IF NOT EXISTS idx_template_matches_source ON template_matches(source_path); +CREATE INDEX IF NOT EXISTS idx_correction_signals_source ON correction_signals(source_path); +` diff --git a/internal/storage/migration_plan_test.go b/internal/storage/migration_plan_test.go new file mode 100644 index 0000000..4af308b --- /dev/null +++ b/internal/storage/migration_plan_test.go @@ -0,0 +1,1522 @@ +package storage + +import ( + "context" + "crypto/sha256" + "database/sql" + "fmt" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/pablontiv/backscroll/internal/compat" +) + +func TestCatalogGoLineagesUpgradeLosslessly(t *testing.T) { + catalog, err := compat.LoadCatalog() + if err != nil { + t.Fatal(err) + } + + seen := map[string]bool{} + var fixtures []struct { + name string + appliedVersion int + expectSnapshot bool + hasSourceMetadata bool + } + for _, release := range catalog.Releases { + if seen[release.Fixture] { + continue + } + seen[release.Fixture] = true + fixtures = append(fixtures, struct { + name string + appliedVersion int + expectSnapshot bool + hasSourceMetadata bool + }{ + name: release.Fixture, + appliedVersion: release.AppliedVersion, + expectSnapshot: fixtureNeedsDestructiveSnapshot(release.AppliedVersion, release.HasSourceMetadata), + hasSourceMetadata: release.HasSourceMetadata, + }) + } + for _, fixture := range catalog.UnmanifestedFixtures { + if seen[fixture.Fixture] { + continue + } + seen[fixture.Fixture] = true + fixtures = append(fixtures, struct { + name string + appliedVersion int + expectSnapshot bool + hasSourceMetadata bool + }{ + name: fixture.Fixture, + appliedVersion: fixture.AppliedVersion, + expectSnapshot: fixtureNeedsDestructiveSnapshot(fixture.AppliedVersion, fixture.HasSourceMetadata), + hasSourceMetadata: fixture.HasSourceMetadata, + }) + } + + for _, fixture := range fixtures { + t.Run(fixture.name, func(t *testing.T) { + dbPath := createFixtureDatabase(t, fixture.name) + want := seedFixtureSentinels(t, dbPath) + + db, diag, err := OpenCompatible(context.Background(), dbPath) + if err != nil || diag != nil { + t.Fatalf("open compatible error=%v diagnostic=%+v", err, diag) + } + defer func() { _ = db.Close() }() + + assertSearchItems(t, db.DB(), want.SearchItems) + assertSearchItemsByUUID(t, db.DB(), want.ToolSearchItems) + assertTableSentinels(t, db.DB(), want) + assertFTSQueryable(t, db.DB(), "sentinelterm", len(want.SearchItems)) + assertFTSQueryable(t, db.DB(), "sentinelcmd", 0) + assertToolFTSQueryable(t, db.DB(), "sentinelcmd", len(want.ToolSearchItems)) + assertCurrentShape(t, db.DB()) + if got, wantRows := loadStorageMigrationRows(t, db.DB()), authoritativeCurrentMigrationRows(); !reflect.DeepEqual(got, wantRows) { + t.Fatalf("schema_migrations rows differ\ngot: %+v\nwant: %+v", got, wantRows) + } + + snapshotPath := maybeOnlySnapshot(t, dbPath) + if fixture.expectSnapshot && snapshotPath == "" { + t.Fatalf("expected snapshot for fixture %s with destructive migration steps", fixture.name) + } + if !fixture.expectSnapshot && snapshotPath != "" { + t.Fatalf("unexpected snapshot for fixture %s: %s", fixture.name, snapshotPath) + } + if snapshotPath != "" { + snapshot, err := OpenReadOnly(snapshotPath) + if err != nil { + t.Fatalf("open snapshot read-only: %v", err) + } + defer func() { _ = snapshot.Close() }() + assertSearchItems(t, snapshot.DB(), want.SearchItems) + if fixture.appliedVersion < 5 { + assertTableExists(t, snapshot.DB(), "session_events", true) + } + if fixture.hasSourceMetadata { + assertColumnExists(t, snapshot.DB(), "search_items", "source_metadata", true) + } + } + }) + } +} + +func TestHistoricalLineageWithoutSourceMetadataUpgradesLosslessly(t *testing.T) { + dbPath := createFixtureDatabase(t, "v3-no-source-metadata.sql") + wantRows := seedMigrationSentinels(t, dbPath) + + db, diag, err := OpenCompatible(context.Background(), dbPath) + if err != nil || diag != nil { + t.Fatalf("open compatible error=%v diagnostic=%+v", err, diag) + } + defer func() { _ = db.Close() }() + + assertSearchItems(t, db.DB(), wantRows) + assertChunkCount(t, db.DB(), 1) + assertFTSQueryable(t, db.DB(), "sentinelterm", len(wantRows)) + assertCurrentShape(t, db.DB()) +} + +func TestMigrationSnapshotAndRollbackOnDestructiveFailure(t *testing.T) { + ctx := context.Background() + dbPath := createFixtureDatabase(t, "v3-no-source-metadata.sql") + wantRows := seedMigrationSentinels(t, dbPath) + + db, err := openWithoutSetup(dbPath) + if err != nil { + t.Fatalf("open without setup: %v", err) + } + defer func() { _ = db.Close() }() + + inspect, err := OpenReadOnly(dbPath) + if err != nil { + t.Fatalf("open inspect: %v", err) + } + plan, diag, err := compat.InspectIndex(ctx, inspect.DB()) + _ = inspect.Close() + if err != nil || diag != nil { + t.Fatalf("inspect error=%v diagnostic=%+v", err, diag) + } + plan.Steps = []compat.MigrationStep{ + {Version: 4, Name: "V4 tool_fts trigram index"}, + {Version: 5, Name: "V5 drop phantom session_events"}, + {Version: 6, Name: "V6 drop source_metadata when present"}, + } + if err := db.ApplyMigrationPlan(ctx, plan); err == nil { + t.Fatal("expected destructive migration plan to fail on missing source_metadata") + } + + assertSearchItems(t, db.DB(), wantRows) + assertTableExists(t, db.DB(), "session_events", true) + assertTableExists(t, db.DB(), "tool_fts", false) + assertMigrationVersionCount(t, db.DB(), 4, 0) + + snapshotPath := onlySnapshot(t, dbPath) + snapshot, err := OpenReadOnly(snapshotPath) + if err != nil { + t.Fatalf("open snapshot read-only: %v", err) + } + defer func() { _ = snapshot.Close() }() + assertSearchItems(t, snapshot.DB(), wantRows) + assertTableExists(t, snapshot.DB(), "session_events", true) +} + +func TestOpenCompatibleClosesAndClearsDatabaseOnMigrationError(t *testing.T) { + dbPath := createFixtureDatabase(t, "v3.sql") + migrationErr := fmt.Errorf("injected migration failure") + originalApply := openCompatibleApplyMigrationPlan + openCompatibleApplyMigrationPlan = func(_ *Database, _ context.Context, _ compat.MigrationPlan) error { + return migrationErr + } + t.Cleanup(func() { openCompatibleApplyMigrationPlan = originalApply }) + + db, diag, err := OpenCompatible(context.Background(), dbPath) + if err == nil { + t.Fatal("expected migration error") + } + if err != migrationErr { + t.Fatalf("error = %v, want injected migration error", err) + } + if diag != nil { + t.Fatalf("diagnostic = %+v, want nil", diag) + } + if db != nil { + if pingErr := db.DB().Ping(); pingErr == nil { + _ = db.Close() + t.Fatal("OpenCompatible returned an open database after migration error") + } + t.Fatalf("db = %+v, want nil after migration error", db) + } +} + +func TestOpenCompatibleCreatesMissingDatabase(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "new.db") + db, diag, err := OpenCompatible(context.Background(), dbPath) + if err != nil || diag != nil { + t.Fatalf("open compatible missing error=%v diagnostic=%+v", err, diag) + } + defer func() { _ = db.Close() }() + assertCurrentShape(t, db.DB()) +} + +func TestOpenUsesOrdinaryDeferredTransactions(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "ordinary.db") + db, err := Open(dbPath) + if err != nil { + t.Fatalf("open: %v", err) + } + defer func() { _ = db.Close() }() + assertDatabaseBeginLeavesWriteReservationAvailable(t, dbPath, db.DB()) +} + +func TestOpenCompatibleReturnsOrdinaryHandleWithoutMigration(t *testing.T) { + dbPath := createFixtureDatabase(t, "v13.sql") + db, diag, err := OpenCompatible(context.Background(), dbPath) + if err != nil || diag != nil { + t.Fatalf("open compatible error=%v diagnostic=%+v", err, diag) + } + defer func() { _ = db.Close() }() + assertDatabaseBeginLeavesWriteReservationAvailable(t, dbPath, db.DB()) +} + +func TestOpenCompatibleReturnsOrdinaryHandleAfterMigration(t *testing.T) { + dbPath := createFixtureDatabase(t, "v7.sql") + db, diag, err := OpenCompatible(context.Background(), dbPath) + if err != nil || diag != nil { + t.Fatalf("open compatible error=%v diagnostic=%+v", err, diag) + } + defer func() { _ = db.Close() }() + assertDatabaseBeginLeavesWriteReservationAvailable(t, dbPath, db.DB()) +} + +func TestOpenCompatibleMigrationTransactionReservesWriteLock(t *testing.T) { + dbPath := createFixtureDatabase(t, "v7.sql") + originalBegin := beginMigrationTx + var checked bool + beginMigrationTx = func(ctx context.Context, raw *sql.DB) (*sql.Tx, error) { + tx, err := originalBegin(ctx, raw) + if err != nil { + return nil, err + } + checked = true + assertCompetingBeginImmediateBlocked(t, dbPath) + return tx, nil + } + t.Cleanup(func() { beginMigrationTx = originalBegin }) + + db, diag, err := OpenCompatible(context.Background(), dbPath) + if err != nil || diag != nil { + t.Fatalf("open compatible error=%v diagnostic=%+v", err, diag) + } + defer func() { _ = db.Close() }() + if !checked { + t.Fatal("migration transaction was not observed") + } +} + +func TestSnapshotDatabaseUsesAvailableSiblingName(t *testing.T) { + dbPath := createFixtureDatabase(t, "v13.sql") + if err := os.WriteFile(dbPath+".snapshot", []byte("occupied"), 0o644); err != nil { + t.Fatal(err) + } + snapshotPath, err := SnapshotDatabase(context.Background(), dbPath) + if err != nil { + t.Fatalf("snapshot database: %v", err) + } + if snapshotPath != dbPath+".snapshot.1" { + t.Fatalf("snapshot path = %q, want numbered sibling", snapshotPath) + } + snapshot, err := OpenReadOnly(snapshotPath) + if err != nil { + t.Fatalf("open snapshot read-only: %v", err) + } + defer func() { _ = snapshot.Close() }() + assertCurrentShape(t, snapshot.DB()) +} + +func TestApplyMigrationPlanRejectsUnknownStep(t *testing.T) { + dbPath := createFixtureDatabase(t, "v13.sql") + db, err := openWithoutSetup(dbPath) + if err != nil { + t.Fatalf("open without setup: %v", err) + } + defer func() { _ = db.Close() }() + if err := db.ApplyMigrationPlan(context.Background(), compat.MigrationPlan{Steps: []compat.MigrationStep{{Version: 99, Name: "unknown"}}}); err == nil { + t.Fatal("expected unknown migration step to fail") + } + if err := db.ApplyMigrationPlan(context.Background(), compat.MigrationPlan{}); err != nil { + t.Fatalf("empty migration plan: %v", err) + } +} + +func TestApplyMigrationPlanFromEmptySchemaCreatesCurrentShape(t *testing.T) { + ctx := context.Background() + dbPath := filepath.Join(t.TempDir(), "backscroll.db") + raw, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatal(err) + } + if _, err := raw.Exec(` + CREATE TABLE schema_migrations ( + version INTEGER PRIMARY KEY, + name TEXT NOT NULL, + applied_on TEXT NOT NULL, + checksum TEXT NOT NULL + ) + `); err != nil { + _ = raw.Close() + t.Fatal(err) + } + _ = raw.Close() + + db, err := openWithoutSetup(dbPath) + if err != nil { + t.Fatalf("open without setup: %v", err) + } + defer func() { _ = db.Close() }() + + plan, _, err := compat.InspectIndex(ctx, db.DB()) + if err != nil { + t.Fatalf("inspect empty schema: %v", err) + } + plan.Steps = []compat.MigrationStep{ + {Version: 1, Name: "V1 core schema"}, + {Version: 2, Name: "V2 embedding tables"}, + {Version: 3, Name: "V3 embedding blob column"}, + {Version: 4, Name: "V4 tool_fts trigram index"}, + {Version: 5, Name: "V5 drop phantom session_events"}, + {Version: 6, Name: "V6 drop source_metadata when present"}, + {Version: 7, Name: "V7 reasoning triggers"}, + {Version: 8, Name: "V8 perennity: extraction_version, was_interrupted, tool_events"}, + {Version: 9, Name: "V9 tool_events uuid uniqueness index"}, + {Version: 10, Name: "V10 template mining: message_templates, template_matches"}, + {Version: 11, Name: "V11 correction detection: correction_signals"}, + {Version: 12, Name: "V12 agent classification: annotations"}, + {Version: 13, Name: "V13 backfill discovery indexes"}, + } + if err := db.ApplyMigrationPlan(ctx, plan); err != nil { + t.Fatalf("apply full plan: %v", err) + } + assertCurrentShape(t, db.DB()) +} + +func TestV9HistoricalMigrationDefinitionIsImmutable(t *testing.T) { + checksum := fmt.Sprintf("%x", sha256.Sum256([]byte(sqlV9))) + if checksum != "b16094805a4e08f6e0dd56bce5266c7c5fd71934389da9d13c4132076e546ca2" { + t.Fatalf("sqlV9 checksum = %s, want published historical checksum", checksum) + } + for _, fixture := range []string{"v9.sql", "v10.sql", "v11.sql", "v12.sql", "v13.sql"} { + data, err := os.ReadFile(filepath.Join("..", "compat", "testdata", "release-schemas", fixture)) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(data), "'b16094805a4e08f6e0dd56bce5266c7c5fd71934389da9d13c4132076e546ca2'") { + t.Fatalf("%s does not preserve the published V9 ledger checksum", fixture) + } + if strings.Contains(string(data), "'a84857185adb25a812c78f05b1ba4ee4d28e52b52b28812e077c03e7ae539917'") { + t.Fatalf("%s contains the replacement V9 checksum", fixture) + } + } +} + +func TestSetupSchemaMigrationLedgerMatchesAuthoritativeDefinitions(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "backscroll.db") + db, err := Open(dbPath) + if err != nil { + t.Fatal(err) + } + defer func() { _ = db.Close() }() + + got := loadStorageMigrationRows(t, db.DB()) + want := authoritativeCurrentMigrationRows() + if !reflect.DeepEqual(got, want) { + t.Fatalf("schema_migrations rows differ\ngot: %+v\nwant: %+v", got, want) + } +} + +func TestMigrationFinalShapeFailureRollsBack(t *testing.T) { + ctx := context.Background() + dbPath := createFixtureDatabase(t, "v3.sql") + wantRows := seedMigrationSentinels(t, dbPath) + + inspect, err := OpenReadOnly(dbPath) + if err != nil { + t.Fatalf("open inspect: %v", err) + } + plan, diag, err := compat.InspectIndex(ctx, inspect.DB()) + _ = inspect.Close() + if err != nil || diag != nil { + t.Fatalf("inspect error=%v diagnostic=%+v", err, diag) + } + if len(plan.Steps) < 2 { + t.Fatalf("plan too short: %+v", plan.Steps) + } + plan.Steps = plan.Steps[:len(plan.Steps)-1] + + db, err := openWithoutSetup(dbPath) + if err != nil { + t.Fatalf("open without setup: %v", err) + } + defer func() { _ = db.Close() }() + + if err := db.ApplyMigrationPlan(ctx, plan); err == nil { + t.Fatal("expected final shape verification failure") + } + + assertSearchItems(t, db.DB(), wantRows) + assertMigrationVersionCount(t, db.DB(), 4, 0) + assertTableExists(t, db.DB(), "tool_fts", false) +} + +func TestDestructiveV8AndV9EntryPointsCreateSnapshot(t *testing.T) { + for _, fixture := range []string{"v7.sql", "v8.sql"} { + t.Run(fixture, func(t *testing.T) { + dbPath := createFixtureDatabase(t, fixture) + if fixture == "v8.sql" { + seedToolEvent(t, dbPath, "v8-snapshot-uuid", "/v8-snapshot.jsonl", 1, "Bash", "echo ok", 0, 0, 8) + } + db, diag, err := OpenCompatible(context.Background(), dbPath) + if err != nil || diag != nil { + t.Fatalf("open compatible error=%v diagnostic=%+v", err, diag) + } + defer func() { _ = db.Close() }() + if snapshotPath := maybeOnlySnapshot(t, dbPath); snapshotPath == "" { + t.Fatalf("expected snapshot before destructive migration from %s", fixture) + } + }) + } +} + +func TestApplyMigrationPlanSnapshotsBeforeBeginningTransaction(t *testing.T) { + ctx := context.Background() + dbPath := createFixtureDatabase(t, "v7.sql") + inspect, err := OpenReadOnly(dbPath) + if err != nil { + t.Fatal(err) + } + plan, diag, err := compat.InspectIndex(ctx, inspect.DB()) + _ = inspect.Close() + if err != nil || diag != nil { + t.Fatalf("inspect error=%v diagnostic=%+v", err, diag) + } + + db, err := openWithoutSetup(dbPath) + if err != nil { + t.Fatal(err) + } + defer func() { _ = db.Close() }() + + originalSnapshot := snapshotDatabase + originalBegin := beginMigrationTx + var order []string + snapshotDatabase = func(context.Context, string) (string, error) { + order = append(order, "snapshot") + return dbPath + ".snapshot", nil + } + beginMigrationTx = func(ctx context.Context, raw *sql.DB) (*sql.Tx, error) { + order = append(order, "begin") + return raw.BeginTx(ctx, nil) + } + t.Cleanup(func() { + snapshotDatabase = originalSnapshot + beginMigrationTx = originalBegin + }) + + if err := db.ApplyMigrationPlan(ctx, plan); err != nil { + t.Fatalf("apply migration plan: %v", err) + } + if !reflect.DeepEqual(order[:2], []string{"snapshot", "begin"}) { + t.Fatalf("snapshot/transaction order = %+v, want snapshot before begin", order) + } +} + +func TestV9DuplicateToolEventsConflictAbortsBeforeMutation(t *testing.T) { + ctx := context.Background() + dbPath := createFixtureDatabase(t, "v8.sql") + seedToolEvent(t, dbPath, "dup-tool-uuid", "/dup-a.jsonl", 1, "Bash", "echo one", 0, 0, 8) + seedToolEvent(t, dbPath, "dup-tool-uuid", "/dup-b.jsonl", 2, "Bash", "echo two", 0, 0, 8) + + inspect, err := OpenReadOnly(dbPath) + if err != nil { + t.Fatal(err) + } + plan, diag, err := compat.InspectIndex(ctx, inspect.DB()) + _ = inspect.Close() + if err != nil || diag != nil { + t.Fatalf("inspect error=%v diagnostic=%+v", err, diag) + } + + db, err := openWithoutSetup(dbPath) + if err != nil { + t.Fatal(err) + } + defer func() { _ = db.Close() }() + var snapshotCalled, beginCalled bool + originalSnapshot := snapshotDatabase + originalBegin := beginMigrationTx + snapshotDatabase = func(ctx context.Context, path string) (string, error) { + snapshotCalled = true + return originalSnapshot(ctx, path) + } + beginMigrationTx = func(ctx context.Context, raw *sql.DB) (*sql.Tx, error) { + beginCalled = true + return raw.BeginTx(ctx, nil) + } + t.Cleanup(func() { + snapshotDatabase = originalSnapshot + beginMigrationTx = originalBegin + }) + + err = db.ApplyMigrationPlan(ctx, plan) + if err == nil { + t.Fatal("expected conflicting duplicate tool_events to abort V9") + } + if !strings.Contains(err.Error(), "conflicting tool_events") { + t.Fatalf("error = %v, want conflicting tool_events", err) + } + if snapshotCalled || beginCalled { + t.Fatalf("V9 duplicate conflict reached snapshot=%v begin=%v; want preflight abort before both", snapshotCalled, beginCalled) + } + assertToolEventUUIDCount(t, db.DB(), "dup-tool-uuid", 2) + assertMigrationVersionCount(t, db.DB(), 9, 0) + if snapshotPath := maybeOnlySnapshot(t, dbPath); snapshotPath != "" { + t.Fatalf("unexpected snapshot for V9 preflight failure: %s", snapshotPath) + } +} + +func TestV9DuplicateToolEventsDifferingOnlyProvenanceAbortBeforeSnapshotOrTransaction(t *testing.T) { + ctx := context.Background() + dbPath := createFixtureDatabase(t, "v8.sql") + seedToolEvent(t, dbPath, "provenance-tool-uuid", "/dup-a.jsonl", 1, "Bash", "echo same", 0, 0, 8) + seedToolEvent(t, dbPath, "provenance-tool-uuid", "/dup-b.jsonl", 2, "Bash", "echo same", 0, 0, 8) + + inspect, err := OpenReadOnly(dbPath) + if err != nil { + t.Fatal(err) + } + plan, diag, err := compat.InspectIndex(ctx, inspect.DB()) + _ = inspect.Close() + if err != nil || diag != nil { + t.Fatalf("inspect error=%v diagnostic=%+v", err, diag) + } + + db, err := openWithoutSetup(dbPath) + if err != nil { + t.Fatal(err) + } + defer func() { _ = db.Close() }() + var snapshotCalled, beginCalled bool + originalSnapshot := snapshotDatabase + originalBegin := beginMigrationTx + snapshotDatabase = func(ctx context.Context, path string) (string, error) { + snapshotCalled = true + return originalSnapshot(ctx, path) + } + beginMigrationTx = func(ctx context.Context, raw *sql.DB) (*sql.Tx, error) { + beginCalled = true + return raw.BeginTx(ctx, nil) + } + t.Cleanup(func() { + snapshotDatabase = originalSnapshot + beginMigrationTx = originalBegin + }) + + err = db.ApplyMigrationPlan(ctx, plan) + if err == nil { + t.Fatal("expected provenance-different duplicate tool_events to abort V9") + } + if !strings.Contains(err.Error(), "conflicting tool_events") { + t.Fatalf("error = %v, want conflicting tool_events", err) + } + if snapshotCalled || beginCalled { + t.Fatalf("V9 provenance conflict reached snapshot=%v begin=%v; want preflight abort before both", snapshotCalled, beginCalled) + } + assertToolEventUUIDCount(t, db.DB(), "provenance-tool-uuid", 2) + assertMigrationVersionCount(t, db.DB(), 9, 0) + if snapshotPath := maybeOnlySnapshot(t, dbPath); snapshotPath != "" { + t.Fatalf("unexpected snapshot for V9 preflight failure: %s", snapshotPath) + } +} + +func TestV9ExactDuplicatePreflightIsReadOnlyAndAllowsMigrationSQLToCollapse(t *testing.T) { + ctx := context.Background() + db, err := sql.Open("sqlite", ":memory:") + if err != nil { + t.Fatal(err) + } + defer db.Close() + if _, err := db.Exec(` + CREATE TABLE search_items ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source TEXT NOT NULL DEFAULT 'session', + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + role TEXT NOT NULL, + text TEXT NOT NULL, + timestamp TEXT, + uuid TEXT UNIQUE, + project TEXT, + content_type TEXT NOT NULL DEFAULT 'text', + extraction_version INTEGER, + was_interrupted INTEGER + ); + CREATE TABLE tool_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + message_uuid TEXT, + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + tool_name TEXT NOT NULL, + command_head TEXT, + is_error INTEGER, + exit_code INTEGER, + extraction_version INTEGER NOT NULL + ); + INSERT INTO search_items (source, source_path, ordinal, role, text, timestamp, uuid, project, content_type, extraction_version, was_interrupted) + VALUES ('session', '/same.jsonl', 7, 'assistant', 'tool payload', '2026-08-18T00:00:00Z', 'exact-tool-uuid', 'project-a', 'tool', 8, 0); + INSERT INTO tool_events (message_uuid, source_path, ordinal, tool_name, command_head, is_error, exit_code, extraction_version) + VALUES ('exact-tool-uuid', '/same.jsonl', 7, 'Bash', 'echo same', 0, 0, 8), + ('exact-tool-uuid', '/same.jsonl', 7, 'Bash', 'echo same', 0, 0, 8); + `); err != nil { + t.Fatal(err) + } + tx, err := db.BeginTx(ctx, nil) + if err != nil { + t.Fatal(err) + } + defer func() { _ = tx.Rollback() }() + if err := prepareV9ToolEventDuplicates(ctx, tx); err != nil { + t.Fatalf("exact duplicate preflight rejected rows: %v", err) + } + var count int + if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM tool_events WHERE message_uuid = 'exact-tool-uuid'`).Scan(&count); err != nil { + t.Fatal(err) + } + if count != 2 { + t.Fatalf("preflight changed exact duplicate count to %d, want 2", count) + } + if _, err := tx.ExecContext(ctx, sqlV9); err != nil { + t.Fatalf("historical V9 SQL should collapse exact duplicates and create uniqueness index: %v", err) + } + if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM tool_events WHERE message_uuid = 'exact-tool-uuid'`).Scan(&count); err != nil { + t.Fatal(err) + } + if count != 1 { + t.Fatalf("historical V9 SQL retained %d exact duplicates, want 1", count) + } +} + +func TestV9StructuralDuplicateComparisonRejectsEmbeddedNULSerializationCollision(t *testing.T) { + ctx := context.Background() + db, err := sql.Open("sqlite", ":memory:") + if err != nil { + t.Fatal(err) + } + defer db.Close() + if _, err := db.Exec(` + CREATE TABLE search_items ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source TEXT NOT NULL DEFAULT 'session', + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + role TEXT NOT NULL, + text TEXT NOT NULL, + timestamp TEXT, + uuid TEXT UNIQUE, + project TEXT, + content_type TEXT NOT NULL DEFAULT 'text', + extraction_version INTEGER, + was_interrupted INTEGER + ); + CREATE TABLE tool_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + message_uuid TEXT, + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + tool_name TEXT NOT NULL, + command_head TEXT, + is_error INTEGER, + exit_code INTEGER, + extraction_version INTEGER NOT NULL + ); + INSERT INTO tool_events (message_uuid, source_path, ordinal, tool_name, command_head, is_error, exit_code, extraction_version) + VALUES ('nul-collision-tool-uuid', '/same.jsonl', 7, 'tool', 'cmd' || char(0) || '', NULL, 0, 8), + ('nul-collision-tool-uuid', '/same.jsonl', 7, 'tool' || char(0) || 's:cmd', NULL, NULL, 0, 8); + `); err != nil { + t.Fatal(err) + } + + err = prepareV9ToolEventDuplicates(ctx, db) + if err == nil { + t.Fatal("expected embedded-NUL structural difference to abort V9 preflight") + } + if !strings.Contains(err.Error(), "conflicting tool_events") { + t.Fatalf("error = %v, want conflicting tool_events", err) + } +} + +func TestV9StructuralDuplicateComparisonPreservesNullableDistinctions(t *testing.T) { + ctx := context.Background() + db, err := sql.Open("sqlite", ":memory:") + if err != nil { + t.Fatal(err) + } + defer db.Close() + if _, err := db.Exec(` + CREATE TABLE tool_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + message_uuid TEXT, + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + tool_name TEXT NOT NULL, + command_head TEXT, + is_error INTEGER, + exit_code INTEGER, + extraction_version INTEGER NOT NULL + ); + INSERT INTO tool_events (message_uuid, source_path, ordinal, tool_name, command_head, is_error, exit_code, extraction_version) + VALUES ('nullable-tool-uuid', '/same.jsonl', 7, 'Bash', '', NULL, NULL, 8), + ('nullable-tool-uuid', '/same.jsonl', 7, 'Bash', NULL, 0, 0, 8); + `); err != nil { + t.Fatal(err) + } + + err = prepareV9ToolEventDuplicates(ctx, db) + if err == nil { + t.Fatal("expected NULL-vs-empty/zero structural differences to abort V9 preflight") + } + if !strings.Contains(err.Error(), "conflicting tool_events") { + t.Fatalf("error = %v, want conflicting tool_events", err) + } +} + +func TestV9TransactionalRecheckCatchesDuplicateInsertedAfterPreflight(t *testing.T) { + ctx := context.Background() + dbPath := createFixtureDatabase(t, "v8.sql") + seedToolEvent(t, dbPath, "late-duplicate-tool-uuid", "/late-a.jsonl", 1, "Bash", "echo one", 0, 0, 8) + + inspect, err := OpenReadOnly(dbPath) + if err != nil { + t.Fatal(err) + } + plan, diag, err := compat.InspectIndex(ctx, inspect.DB()) + _ = inspect.Close() + if err != nil || diag != nil { + t.Fatalf("inspect error=%v diagnostic=%+v", err, diag) + } + + db, err := openWithoutSetup(dbPath) + if err != nil { + t.Fatal(err) + } + defer func() { _ = db.Close() }() + originalSnapshot := snapshotDatabase + var injected bool + snapshotDatabase = func(ctx context.Context, path string) (string, error) { + seedToolEvent(t, path, "late-duplicate-tool-uuid", "/late-b.jsonl", 2, "Bash", "echo two", 0, 0, 8) + injected = true + return originalSnapshot(ctx, path) + } + t.Cleanup(func() { snapshotDatabase = originalSnapshot }) + + err = db.ApplyMigrationPlan(ctx, plan) + if err == nil { + t.Fatal("expected transactional V9 recheck to abort late duplicate") + } + if !injected { + t.Fatal("late duplicate was not injected after preflight") + } + if !strings.Contains(err.Error(), "conflicting tool_events") { + t.Fatalf("error = %v, want conflicting tool_events", err) + } + raw, openErr := openWithoutSetup(dbPath) + if openErr != nil { + t.Fatal(openErr) + } + defer func() { _ = raw.Close() }() + assertToolEventUUIDCount(t, raw.DB(), "late-duplicate-tool-uuid", 2) + assertMigrationVersionCount(t, raw.DB(), 9, 0) + assertIndexExists(t, raw.DB(), "idx_tool_events_uuid_unique", false) +} + +func TestV9SamePayloadDifferentSourceIsNotAnExactDuplicate(t *testing.T) { + ctx := context.Background() + dbPath := createFixtureDatabase(t, "v8.sql") + seedToolEvent(t, dbPath, "same-payload-tool-uuid", "/exact-a.jsonl", 1, "Bash", "echo same", 0, 0, 8) + seedToolEvent(t, dbPath, "same-payload-tool-uuid", "/exact-b.jsonl", 2, "Bash", "echo same", 0, 0, 8) + + db, diag, err := OpenCompatible(ctx, dbPath) + if err == nil { + if db != nil { + _ = db.Close() + } + t.Fatal("expected same-payload different-source duplicate tool_events to abort V9") + } + if diag != nil { + t.Fatalf("diagnostic = %+v, want migration error", diag) + } + if !strings.Contains(err.Error(), "conflicting tool_events") { + t.Fatalf("error = %v, want conflicting tool_events", err) + } + raw, openErr := openWithoutSetup(dbPath) + if openErr != nil { + t.Fatal(openErr) + } + defer func() { _ = raw.Close() }() + assertToolEventUUIDCount(t, raw.DB(), "same-payload-tool-uuid", 2) + if snapshotPath := maybeOnlySnapshot(t, dbPath); snapshotPath != "" { + t.Fatalf("unexpected snapshot for V9 preflight failure: %s", snapshotPath) + } +} + +func TestApplyMigrationPlanUsesReceiverPathForSnapshotAndVerification(t *testing.T) { + ctx := context.Background() + receiverPath := createFixtureDatabase(t, "v3.sql") + unrelatedPath := createFixtureDatabase(t, "v13.sql") + + inspect, err := OpenReadOnly(receiverPath) + if err != nil { + t.Fatal(err) + } + plan, diag, err := compat.InspectIndex(ctx, inspect.DB()) + _ = inspect.Close() + if err != nil || diag != nil { + t.Fatalf("inspect error=%v diagnostic=%+v", err, diag) + } + db, err := openWithoutSetup(receiverPath) + if err != nil { + t.Fatal(err) + } + defer func() { _ = db.Close() }() + + if err := db.ApplyMigrationPlan(ctx, plan); err != nil { + t.Fatalf("apply migration plan: %v", err) + } + if snapshotPath := maybeOnlySnapshot(t, receiverPath); snapshotPath == "" { + t.Fatal("receiver database was not snapshotted") + } + if snapshotPath := maybeOnlySnapshot(t, unrelatedPath); snapshotPath != "" { + t.Fatalf("unrelated path was snapshotted: %s", snapshotPath) + } + assertCurrentShape(t, db.DB()) +} + +func TestApplyMigrationPlanRechecksShapeInsideTransaction(t *testing.T) { + ctx := context.Background() + dbPath := createFixtureDatabase(t, "v7.sql") + inspect, err := OpenReadOnly(dbPath) + if err != nil { + t.Fatal(err) + } + plan, diag, err := compat.InspectIndex(ctx, inspect.DB()) + _ = inspect.Close() + if err != nil || diag != nil { + t.Fatalf("inspect error=%v diagnostic=%+v", err, diag) + } + mutateFixtureDB(t, dbPath, `CREATE TABLE shape_changed_after_planning (id INTEGER PRIMARY KEY);`) + + db, err := openWithoutSetup(dbPath) + if err != nil { + t.Fatal(err) + } + defer func() { _ = db.Close() }() + err = db.ApplyMigrationPlan(ctx, plan) + if err == nil { + t.Fatal("expected shape mismatch to abort") + } + if !strings.Contains(err.Error(), "changed since inspection") { + t.Fatalf("error = %v, want changed since inspection", err) + } + assertMigrationVersionCount(t, db.DB(), 8, 0) + assertTableExists(t, db.DB(), "shape_changed_after_planning", true) +} + +type storageMigrationRow struct { + Version int + Name string + Checksum string +} + +type sentinelSearchItem struct { + ID int64 + Source string + SourcePath string + Ordinal int + Role string + Text string + Timestamp string + UUID string + Project string + ContentType string +} + +func assertDatabaseBeginLeavesWriteReservationAvailable(t *testing.T, dbPath string, db *sql.DB) { + t.Helper() + tx, err := db.Begin() + if err != nil { + t.Fatalf("begin ordinary transaction: %v", err) + } + defer func() { _ = tx.Rollback() }() + + competing, err := sql.Open("sqlite", dbPath+"?_pragma=busy_timeout(0)") + if err != nil { + t.Fatalf("open competing connection: %v", err) + } + defer func() { _ = competing.Close() }() + if _, err := competing.Exec("BEGIN IMMEDIATE"); err != nil { + t.Fatalf("ordinary BEGIN reserved the write lock; competing BEGIN IMMEDIATE failed: %v", err) + } + if _, err := competing.Exec("ROLLBACK"); err != nil { + t.Fatalf("rollback competing transaction: %v", err) + } +} + +func assertCompetingBeginImmediateBlocked(t *testing.T, dbPath string) { + t.Helper() + competing, err := sql.Open("sqlite", dbPath+"?_pragma=busy_timeout(0)") + if err != nil { + t.Fatalf("open competing connection: %v", err) + } + defer func() { _ = competing.Close() }() + if _, err := competing.Exec("BEGIN IMMEDIATE"); err == nil { + _, _ = competing.Exec("ROLLBACK") + t.Fatal("migration transaction did not reserve the write lock; competing BEGIN IMMEDIATE succeeded") + } +} + +func createFixtureDatabase(t *testing.T, fixture string) string { + t.Helper() + + data, err := os.ReadFile(filepath.Join("..", "compat", "testdata", "release-schemas", fixture)) + if err != nil { + t.Fatal(err) + } + tmp := t.TempDir() + fixtureCopy := filepath.Join(tmp, fixture) + if err := os.WriteFile(fixtureCopy, data, 0o644); err != nil { + t.Fatal(err) + } + + dbPath := filepath.Join(tmp, "backscroll.db") + db, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatal(err) + } + defer db.Close() + fixtureSQL, err := os.ReadFile(fixtureCopy) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(string(fixtureSQL)); err != nil { + t.Fatal(err) + } + return dbPath +} + +func seedMigrationSentinels(t *testing.T, dbPath string) []sentinelSearchItem { + t.Helper() + + db, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + items := []sentinelSearchItem{ + {Source: "session", SourcePath: "/project/session.jsonl", Ordinal: 1, Role: "user", Text: "sentinelterm first message", Timestamp: "2026-08-18T00:00:00Z", UUID: "sentinel-uuid-1", Project: "project", ContentType: "text"}, + {Source: "session", SourcePath: "/project/session.jsonl", Ordinal: 2, Role: "assistant", Text: "sentinelterm second message", Timestamp: "2026-08-18T00:00:00Z", UUID: "sentinel-uuid-2", Project: "project", ContentType: "text"}, + } + for i := range items { + res, err := db.Exec(`INSERT INTO search_items (source, source_path, ordinal, role, text, timestamp, uuid, project, content_type) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, + items[i].Source, items[i].SourcePath, items[i].Ordinal, items[i].Role, items[i].Text, items[i].Timestamp, items[i].UUID, items[i].Project, items[i].ContentType) + if err != nil { + t.Fatal(err) + } + items[i].ID, err = res.LastInsertId() + if err != nil { + t.Fatal(err) + } + } + if tableExists(t, db, "chunks") { + if columnExists(t, db, "chunks", "embedding") { + if _, err := db.Exec(`INSERT INTO chunks (source_id, chunk_idx, content, token_count, created_at, embedding) + VALUES ('sentinel-uuid-1', 0, 'chunk sentinel', 2, 1, X'0102')`); err != nil { + t.Fatal(err) + } + } else { + if _, err := db.Exec(`INSERT INTO chunks (source_id, chunk_idx, content, token_count, created_at) + VALUES ('sentinel-uuid-1', 0, 'chunk sentinel', 2, 1)`); err != nil { + t.Fatal(err) + } + } + } + return items +} + +func assertSearchItems(t *testing.T, db *sql.DB, want []sentinelSearchItem) { + t.Helper() + + rows, err := db.Query(`SELECT id, source, source_path, ordinal, role, text, timestamp, uuid, project, content_type + FROM search_items WHERE uuid LIKE 'sentinel-uuid-%' ORDER BY id`) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + + var got []sentinelSearchItem + for rows.Next() { + var item sentinelSearchItem + if err := rows.Scan(&item.ID, &item.Source, &item.SourcePath, &item.Ordinal, &item.Role, &item.Text, &item.Timestamp, &item.UUID, &item.Project, &item.ContentType); err != nil { + t.Fatal(err) + } + got = append(got, item) + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("search_items mismatch\ngot: %+v\nwant: %+v", got, want) + } +} + +func assertSearchItemsByUUID(t *testing.T, db *sql.DB, want []sentinelSearchItem) { + t.Helper() + if len(want) == 0 { + return + } + + placeholders := strings.TrimRight(strings.Repeat("?,", len(want)), ",") + args := make([]any, 0, len(want)) + for _, item := range want { + args = append(args, item.UUID) + } + rows, err := db.Query(`SELECT id, source, source_path, ordinal, role, text, timestamp, uuid, project, content_type + FROM search_items WHERE uuid IN (`+placeholders+`) ORDER BY id`, args...) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + + var got []sentinelSearchItem + for rows.Next() { + var item sentinelSearchItem + if err := rows.Scan(&item.ID, &item.Source, &item.SourcePath, &item.Ordinal, &item.Role, &item.Text, &item.Timestamp, &item.UUID, &item.Project, &item.ContentType); err != nil { + t.Fatal(err) + } + got = append(got, item) + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("search_items by uuid mismatch\ngot: %+v\nwant: %+v", got, want) + } +} + +func assertChunkCount(t *testing.T, db *sql.DB, want int) { + t.Helper() + var got int + if err := db.QueryRow(`SELECT COUNT(*) FROM chunks WHERE source_id = 'sentinel-uuid-1'`).Scan(&got); err != nil { + t.Fatal(err) + } + if got != want { + t.Fatalf("chunk count = %d, want %d", got, want) + } +} + +func assertFTSQueryable(t *testing.T, db *sql.DB, term string, want int) { + t.Helper() + var got int + if err := db.QueryRow(`SELECT COUNT(*) FROM messages_fts WHERE messages_fts MATCH ?`, term).Scan(&got); err != nil { + t.Fatal(err) + } + if got != want { + t.Fatalf("messages_fts count = %d, want %d", got, want) + } +} + +func assertToolFTSQueryable(t *testing.T, db *sql.DB, term string, want int) { + t.Helper() + var got int + if err := db.QueryRow(`SELECT COUNT(*) FROM tool_fts WHERE tool_fts MATCH ?`, term).Scan(&got); err != nil { + t.Fatal(err) + } + if got != want { + t.Fatalf("tool_fts count = %d, want %d", got, want) + } +} + +type fixtureSentinels struct { + SearchItems []sentinelSearchItem + ToolSearchItems []sentinelSearchItem + TableRows map[string][][]string + IndexedFiles bool + SessionTags bool + DynamicStopwords bool + Chunks bool + EmbeddingMetadata bool + ToolEvents bool + MessageTemplates bool + TemplateMatches bool + CorrectionSignals bool + Annotations bool +} + +func seedFixtureSentinels(t *testing.T, dbPath string) fixtureSentinels { + t.Helper() + want := fixtureSentinels{} + want.SearchItems = seedMigrationSentinels(t, dbPath) + + db, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + if tableExists(t, db, "indexed_files") { + if _, err := db.Exec(`INSERT INTO indexed_files (path, hash, last_indexed) VALUES ('/sentinel/indexed.jsonl', 'sentinel-hash', '2026-08-18T00:00:00Z')`); err != nil { + t.Fatal(err) + } + want.IndexedFiles = true + } + if tableExists(t, db, "session_tags") { + if _, err := db.Exec(`INSERT INTO session_tags (source_path, tag) VALUES ('/project/session.jsonl', 'sentinel-tag')`); err != nil { + t.Fatal(err) + } + want.SessionTags = true + } + if tableExists(t, db, "dynamic_stopwords") { + if _, err := db.Exec(`INSERT INTO dynamic_stopwords (term) VALUES ('sentinelstopword')`); err != nil { + t.Fatal(err) + } + want.DynamicStopwords = true + } + var chunkID int64 + if tableExists(t, db, "chunks") { + var res sql.Result + var err error + if columnExists(t, db, "chunks", "embedding") { + res, err = db.Exec(`INSERT OR IGNORE INTO chunks (source_id, chunk_idx, content, token_count, created_at, embedding) + VALUES ('sentinel-uuid-2', 0, 'chunk sentinel two', 3, 1, X'0304')`) + } else { + res, err = db.Exec(`INSERT OR IGNORE INTO chunks (source_id, chunk_idx, content, token_count, created_at) + VALUES ('sentinel-uuid-2', 0, 'chunk sentinel two', 3, 1)`) + } + if err != nil { + t.Fatal(err) + } + chunkID, err = res.LastInsertId() + if err != nil { + t.Fatal(err) + } + want.Chunks = true + } + if chunkID != 0 && tableExists(t, db, "embedding_metadata") { + if _, err := db.Exec(`INSERT INTO embedding_metadata (chunk_id, model_name, model_version, dimensions, created_at) + VALUES (?, 'sentinel-model', 'v1', 2, 1)`, chunkID); err != nil { + t.Fatal(err) + } + want.EmbeddingMetadata = true + } + if tableExists(t, db, "search_items") && columnExists(t, db, "search_items", "content_type") { + toolItem := sentinelSearchItem{Source: "session", SourcePath: "/project/tool-session.jsonl", Ordinal: 101, Role: "assistant", Text: "sentinelcmd tool text", Timestamp: "2026-08-18T00:00:00Z", UUID: "tool-fts-sentinel-uuid", Project: "project", ContentType: "tool"} + res, err := db.Exec(`INSERT INTO search_items (source, source_path, ordinal, role, text, timestamp, uuid, project, content_type) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, toolItem.Source, toolItem.SourcePath, toolItem.Ordinal, toolItem.Role, toolItem.Text, toolItem.Timestamp, toolItem.UUID, toolItem.Project, toolItem.ContentType) + if err != nil { + t.Fatal(err) + } + toolItem.ID, err = res.LastInsertId() + if err != nil { + t.Fatal(err) + } + want.ToolSearchItems = append(want.ToolSearchItems, toolItem) + } + if tableExists(t, db, "tool_events") { + seedToolEventOpen(t, db, "sentinel-tool-event-uuid", "/project/tool-session.jsonl", 101, "Bash", "sentinelcmd", 0, 0, 8) + want.ToolEvents = true + } + var templateID int64 + if tableExists(t, db, "message_templates") { + res, err := db.Exec(`INSERT INTO message_templates (signature, normalization_version, template_text, occurrence_count, first_seen, last_seen) + VALUES ('sentinel-template-signature', 1, 'sentinel template text', 1, '2026-08-18T00:00:00Z', '2026-08-18T00:00:00Z')`) + if err != nil { + t.Fatal(err) + } + templateID, err = res.LastInsertId() + if err != nil { + t.Fatal(err) + } + want.MessageTemplates = true + } + if templateID != 0 && tableExists(t, db, "template_matches") { + if _, err := db.Exec(`INSERT INTO template_matches (template_id, item_uuid, source_path, ordinal) + VALUES (?, 'sentinel-uuid-1', '/project/session.jsonl', 1)`, templateID); err != nil { + t.Fatal(err) + } + want.TemplateMatches = true + } + if tableExists(t, db, "correction_signals") { + if _, err := db.Exec(`INSERT INTO correction_signals (item_uuid, source_path, ordinal, detector, confidence, extraction_version) + VALUES ('sentinel-uuid-1', '/project/session.jsonl', 1, 'sentinel-detector', 0.9, 1)`); err != nil { + t.Fatal(err) + } + want.CorrectionSignals = true + } + if tableExists(t, db, "annotations") { + if _, err := db.Exec(`INSERT INTO annotations (item_uuid, source_path, ordinal, kind, label, source, created_at) + VALUES ('sentinel-uuid-1', '/project/session.jsonl', 1, 'sentinel-kind', 'sentinel-label', 'agent', '2026-08-18T00:00:00Z')`); err != nil { + t.Fatal(err) + } + want.Annotations = true + } + want.TableRows = snapshotSentinelTableRows(t, db) + return want +} + +func assertTableSentinels(t *testing.T, db *sql.DB, want fixtureSentinels) { + t.Helper() + for table, wantRows := range want.TableRows { + gotRows := querySentinelTableRows(t, db, table) + if !reflect.DeepEqual(gotRows, wantRows) { + t.Fatalf("sentinel rows for %s differ\ngot: %+v\nwant: %+v", table, gotRows, wantRows) + } + } +} + +func snapshotSentinelTableRows(t *testing.T, db *sql.DB) map[string][][]string { + t.Helper() + result := map[string][][]string{} + for _, table := range []string{ + "indexed_files", + "session_tags", + "dynamic_stopwords", + "chunks", + "embedding_metadata", + "tool_events", + "message_templates", + "template_matches", + "correction_signals", + "annotations", + } { + if !tableExists(t, db, table) { + continue + } + rows := querySentinelTableRows(t, db, table) + if len(rows) > 0 { + result[table] = rows + } + } + return result +} + +func querySentinelTableRows(t *testing.T, db *sql.DB, table string) [][]string { + t.Helper() + chunksQuery := `SELECT CAST(id AS TEXT), source_id, CAST(chunk_idx AS TEXT), content, CAST(token_count AS TEXT), CAST(created_at AS TEXT), '' AS embedding FROM chunks WHERE source_id IN ('sentinel-uuid-1', 'sentinel-uuid-2') ORDER BY id` + if table == "chunks" && columnExists(t, db, "chunks", "embedding") { + chunksQuery = `SELECT CAST(id AS TEXT), source_id, CAST(chunk_idx AS TEXT), content, CAST(token_count AS TEXT), CAST(created_at AS TEXT), CASE WHEN embedding IS NULL THEN '' ELSE hex(embedding) END FROM chunks WHERE source_id IN ('sentinel-uuid-1', 'sentinel-uuid-2') ORDER BY id` + } + queries := map[string]string{ + "indexed_files": `SELECT path, hash, last_indexed FROM indexed_files WHERE path = '/sentinel/indexed.jsonl' ORDER BY path`, + "session_tags": `SELECT source_path, tag FROM session_tags WHERE tag = 'sentinel-tag' ORDER BY source_path, tag`, + "dynamic_stopwords": `SELECT term FROM dynamic_stopwords WHERE term = 'sentinelstopword' ORDER BY term`, + "chunks": chunksQuery, + "embedding_metadata": `SELECT CAST(id AS TEXT), CAST(chunk_id AS TEXT), model_name, model_version, CAST(dimensions AS TEXT), CAST(created_at AS TEXT) FROM embedding_metadata WHERE model_name = 'sentinel-model' ORDER BY id`, + "tool_events": `SELECT CAST(id AS TEXT), message_uuid, source_path, CAST(ordinal AS TEXT), tool_name, COALESCE(command_head, ''), COALESCE(CAST(is_error AS TEXT), ''), COALESCE(CAST(exit_code AS TEXT), ''), CAST(extraction_version AS TEXT) FROM tool_events WHERE message_uuid = 'sentinel-tool-event-uuid' ORDER BY id`, + "message_templates": `SELECT CAST(id AS TEXT), signature, CAST(normalization_version AS TEXT), template_text, CAST(occurrence_count AS TEXT), COALESCE(first_seen, ''), COALESCE(last_seen, '') FROM message_templates WHERE signature = 'sentinel-template-signature' ORDER BY id`, + "template_matches": `SELECT CAST(id AS TEXT), CAST(template_id AS TEXT), COALESCE(item_uuid, ''), source_path, CAST(ordinal AS TEXT) FROM template_matches WHERE item_uuid = 'sentinel-uuid-1' ORDER BY id`, + "correction_signals": `SELECT CAST(id AS TEXT), COALESCE(item_uuid, ''), source_path, CAST(ordinal AS TEXT), detector, CAST(confidence AS TEXT), CAST(extraction_version AS TEXT) FROM correction_signals WHERE detector = 'sentinel-detector' ORDER BY id`, + "annotations": `SELECT CAST(id AS TEXT), COALESCE(item_uuid, ''), source_path, CAST(ordinal AS TEXT), kind, label, source, created_at FROM annotations WHERE kind = 'sentinel-kind' ORDER BY id`, + } + query, ok := queries[table] + if !ok { + t.Fatalf("no sentinel query for %s", table) + } + rows, err := db.Query(query) + if err != nil { + t.Fatalf("query sentinel %s: %v", table, err) + } + defer rows.Close() + columns, err := rows.Columns() + if err != nil { + t.Fatal(err) + } + var result [][]string + for rows.Next() { + values := make([]sql.NullString, len(columns)) + dest := make([]any, len(columns)) + for i := range values { + dest[i] = &values[i] + } + if err := rows.Scan(dest...); err != nil { + t.Fatal(err) + } + row := make([]string, len(columns)) + for i, value := range values { + if value.Valid { + row[i] = value.String + } else { + row[i] = "" + } + } + result = append(result, row) + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + return result +} + +func fixtureNeedsDestructiveSnapshot(appliedVersion int, hasSourceMetadata bool) bool { + if appliedVersion < 5 || (appliedVersion < 6 && hasSourceMetadata) { + return true + } + return appliedVersion < 9 +} + +func tableExists(t *testing.T, db *sql.DB, table string) bool { + t.Helper() + var count int + if err := db.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = ?`, table).Scan(&count); err != nil { + t.Fatal(err) + } + return count == 1 +} + +func assertIndexExists(t *testing.T, db *sql.DB, index string, want bool) { + t.Helper() + var count int + if err := db.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name = ?`, index).Scan(&count); err != nil { + t.Fatal(err) + } + if got := count == 1; got != want { + t.Fatalf("index %s exists = %v, want %v", index, got, want) + } +} + +func assertColumnExists(t *testing.T, db *sql.DB, table, column string, want bool) { + t.Helper() + got := columnExists(t, db, table, column) + if got != want { + t.Fatalf("column %s.%s exists = %v, want %v", table, column, got, want) + } +} + +func columnExists(t *testing.T, db *sql.DB, table, column string) bool { + t.Helper() + rows, err := db.Query(`PRAGMA table_info(` + quoteIdentifierForTest(table) + `)`) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + for rows.Next() { + var cid int + var name, typ string + var notNull, pk int + var defaultValue sql.NullString + if err := rows.Scan(&cid, &name, &typ, ¬Null, &defaultValue, &pk); err != nil { + t.Fatal(err) + } + if name == column { + return true + } + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + return false +} + +func maybeOnlySnapshot(t *testing.T, dbPath string) string { + t.Helper() + matches, err := filepath.Glob(dbPath + ".snapshot*") + if err != nil { + t.Fatal(err) + } + if len(matches) > 1 { + t.Fatalf("snapshot matches = %+v, want at most one", matches) + } + if len(matches) == 0 { + return "" + } + return matches[0] +} + +func quoteIdentifierForTest(identifier string) string { + return `"` + strings.ReplaceAll(identifier, `"`, `""`) + `"` +} + +func seedToolEvent(t *testing.T, dbPath, messageUUID, sourcePath string, ordinal int, toolName, commandHead string, isError, exitCode, extractionVersion int) int64 { + t.Helper() + db, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatal(err) + } + defer db.Close() + return seedToolEventOpen(t, db, messageUUID, sourcePath, ordinal, toolName, commandHead, isError, exitCode, extractionVersion) +} + +func seedToolEventOpen(t *testing.T, db *sql.DB, messageUUID, sourcePath string, ordinal int, toolName, commandHead string, isError, exitCode, extractionVersion int) int64 { + t.Helper() + res, err := db.Exec(`INSERT INTO tool_events (message_uuid, source_path, ordinal, tool_name, command_head, is_error, exit_code, extraction_version) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, messageUUID, sourcePath, ordinal, toolName, commandHead, isError, exitCode, extractionVersion) + if err != nil { + t.Fatal(err) + } + id, err := res.LastInsertId() + if err != nil { + t.Fatal(err) + } + return id +} + +func assertToolEventUUIDCount(t *testing.T, db *sql.DB, messageUUID string, want int) { + t.Helper() + var got int + if err := db.QueryRow(`SELECT COUNT(*) FROM tool_events WHERE message_uuid = ?`, messageUUID).Scan(&got); err != nil { + t.Fatal(err) + } + if got != want { + t.Fatalf("tool_events count for %s = %d, want %d", messageUUID, got, want) + } +} + +func mutateFixtureDB(t *testing.T, dbPath string, stmt string) { + t.Helper() + db, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatal(err) + } + defer db.Close() + if _, err := db.Exec(stmt); err != nil { + t.Fatal(err) + } +} +func loadStorageMigrationRows(t *testing.T, db *sql.DB) []storageMigrationRow { + t.Helper() + + rows, err := db.Query(`SELECT version, name, checksum FROM schema_migrations ORDER BY version`) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + + var result []storageMigrationRow + for rows.Next() { + var row storageMigrationRow + if err := rows.Scan(&row.Version, &row.Name, &row.Checksum); err != nil { + t.Fatal(err) + } + result = append(result, row) + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + return result +} + +func authoritativeCurrentMigrationRows() []storageMigrationRow { + return []storageMigrationRow{ + {Version: 1, Name: "V1 core schema", Checksum: "4e07949ccd3912fb3c0e149be9a2e05fdd51f8cedb8df1f28b3bb5ac5afe532a"}, + {Version: 2, Name: "V2 embedding tables", Checksum: "37dc9627f01f0e2d0fbea6bba5cd9f609d5da05089eeb9541a057dd2290cf8af"}, + {Version: 3, Name: "V3 embedding blob column", Checksum: "36cd183f10ff84ab4753be027078cdd710b46efdc830053d4a641af725006ea5"}, + {Version: 4, Name: "V4 tool_fts trigram index", Checksum: "77e59cf515f33c466282e0f3e921377f584794d0b7cade1704f566374a569a55"}, + {Version: 5, Name: "V5 drop phantom session_events", Checksum: "aedaab81efb6bc34d3f664468b71c1a716f5b55d5132156cb43bd7462d549c7b"}, + {Version: 6, Name: "V6 drop phantom source_metadata column", Checksum: "a327b9b6e7b8f5fe369c9fc08093ac80a87640daa515890af94b269d430e9378"}, + {Version: 7, Name: "V7 reasoning content_type routes to messages_fts", Checksum: "a80704442c2a0084f98e4bc53978364b14d1c6bd3bd99ba6119a2a9ecbd685e7"}, + {Version: 8, Name: "V8 perennity: extraction_version, was_interrupted, tool_events", Checksum: "6853d72ded3bdc775b52507321c31df44bf35e8277719432ca8aa126ee16cec1"}, + {Version: 9, Name: "V9 tool_events uuid uniqueness index", Checksum: "b16094805a4e08f6e0dd56bce5266c7c5fd71934389da9d13c4132076e546ca2"}, + {Version: 10, Name: "V10 template mining: message_templates, template_matches", Checksum: "0e548d0cb6c47147726f943bfc860500ca9a9bc821df0601998876ea5e9652c2"}, + {Version: 11, Name: "V11 correction detection: correction_signals", Checksum: "5a7180f901c5feacb67cd104a61e7b7ba9cec69aaeab1d2b5d723459dc590456"}, + {Version: 12, Name: "V12 agent classification: annotations (free-form labels; enum freeze deferred)", Checksum: "b3fb66fd2924a9f07a3e4ec0ba253fc1d6965664615a795be6b22d01ab292108"}, + {Version: 13, Name: "V13 backfill discovery indexes", Checksum: "2172ce531c670806933ffe3005fdc0a2ebb8eb3f84d2bd0d8fa608dddb5d136e"}, + } +} + +func assertCurrentShape(t *testing.T, db *sql.DB) { + t.Helper() + if err := compat.VerifyCurrentShape(context.Background(), db); err != nil { + t.Fatalf("verify current shape: %v", err) + } +} + +func assertTableExists(t *testing.T, db *sql.DB, table string, want bool) { + t.Helper() + var count int + if err := db.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = ?`, table).Scan(&count); err != nil { + t.Fatal(err) + } + if got := count == 1; got != want { + t.Fatalf("table %s exists = %v, want %v", table, got, want) + } +} + +func assertMigrationVersionCount(t *testing.T, db *sql.DB, version int, want int) { + t.Helper() + var got int + if err := db.QueryRow(`SELECT COUNT(*) FROM schema_migrations WHERE version = ?`, version).Scan(&got); err != nil { + t.Fatal(err) + } + if got != want { + t.Fatalf("migration version %d count = %d, want %d", version, got, want) + } +} + +func onlySnapshot(t *testing.T, dbPath string) string { + t.Helper() + matches, err := filepath.Glob(dbPath + ".snapshot*") + if err != nil { + t.Fatal(err) + } + if len(matches) != 1 { + t.Fatalf("snapshot matches = %+v, want exactly one", matches) + } + return matches[0] +} diff --git a/internal/storage/migrations.go b/internal/storage/migrations.go index bb14e70..c63077a 100644 --- a/internal/storage/migrations.go +++ b/internal/storage/migrations.go @@ -1,9 +1,6 @@ package storage -import ( - "crypto/sha256" - "fmt" -) +import "fmt" // SetupSchema creates the database schema if it doesn't already exist. // It idempotently applies all migrations using the schema_migrations table. @@ -183,143 +180,25 @@ func (d *Database) SetupSchema() error { // applyV1Migration applies version 1 of the schema (all core tables). func (d *Database) applyV1Migration() error { - tx, err := d.db.Begin() - if err != nil { - return fmt.Errorf("begin transaction: %w", err) - } - defer func() { _ = tx.Rollback() }() - - // Define the core DDL for this migration (for checksum) - coreDDL := sqlV1CoreDDL - - // Execute all CREATE TABLE statements (idempotent with IF NOT EXISTS) - if _, err := tx.Exec(sqlV1Core); err != nil { - return fmt.Errorf("create core tables: %w", err) - } - - // Create FTS5 virtual table and vocab view (idempotent) - if _, err := tx.Exec(sqlV1FTS5); err != nil { - return fmt.Errorf("create FTS5 virtual table: %w", err) - } - - // Create triggers (idempotent with IF NOT EXISTS) - if _, err := tx.Exec(sqlV1Triggers); err != nil { - return fmt.Errorf("create triggers: %w", err) - } - - // Compute checksum of the core DDL - checksum := sha256.Sum256([]byte(coreDDL)) - checksumHex := fmt.Sprintf("%x", checksum) - - // Record migration as applied - _, err = tx.Exec(` - INSERT INTO schema_migrations (version, name, applied_on, checksum) - VALUES (1, 'V1 core schema', CURRENT_TIMESTAMP, ?) - `, checksumHex) - if err != nil { - return fmt.Errorf("record migration: %w", err) - } - - if err := tx.Commit(); err != nil { - return fmt.Errorf("commit migration: %w", err) - } - - return nil + return d.applySingleMigration(applyV1) } // applyV2Migration adds tables for the embedding system: chunks and embedding_metadata. func (d *Database) applyV2Migration() error { - tx, err := d.db.Begin() - if err != nil { - return fmt.Errorf("begin transaction: %w", err) - } - defer func() { _ = tx.Rollback() }() - - if _, err := tx.Exec(sqlV2); err != nil { - return fmt.Errorf("create embedding tables: %w", err) - } - - checksum := sha256.Sum256([]byte(sqlV2)) - checksumHex := fmt.Sprintf("%x", checksum) - - _, err = tx.Exec(` - INSERT INTO schema_migrations (version, name, applied_on, checksum) - VALUES (2, 'V2 embedding tables', CURRENT_TIMESTAMP, ?) - `, checksumHex) - if err != nil { - return fmt.Errorf("record migration v2: %w", err) - } - - if err := tx.Commit(); err != nil { - return fmt.Errorf("commit migration v2: %w", err) - } - - return nil + return d.applySingleMigration(applyV2) } // applyV3Migration adds an embedding BLOB column to chunks for pure-Go vector search. // Decision (T039): sqlite-vec requires CGO; we store embedding bytes directly in chunks // and perform cosine similarity in Go (linear scan). func (d *Database) applyV3Migration() error { - tx, err := d.db.Begin() - if err != nil { - return fmt.Errorf("begin transaction: %w", err) - } - defer func() { _ = tx.Rollback() }() - - if _, err := tx.Exec(sqlV3); err != nil { - return fmt.Errorf("add embedding column: %w", err) - } - - checksum := sha256.Sum256([]byte(sqlV3)) - checksumHex := fmt.Sprintf("%x", checksum) - - _, err = tx.Exec(` - INSERT INTO schema_migrations (version, name, applied_on, checksum) - VALUES (3, 'V3 embedding blob column', CURRENT_TIMESTAMP, ?) - `, checksumHex) - if err != nil { - return fmt.Errorf("record migration v3: %w", err) - } - - if err := tx.Commit(); err != nil { - return fmt.Errorf("commit migration v3: %w", err) - } - - return nil + return d.applySingleMigration(applyV3) } // applyV4Migration adds the tool_fts index (trigram tokenizer), branches the // sync triggers by content_type, and repopulates both indexes from search_items. func (d *Database) applyV4Migration() error { - tx, err := d.db.Begin() - if err != nil { - return fmt.Errorf("begin transaction: %w", err) - } - defer func() { _ = tx.Rollback() }() - - if _, err := tx.Exec(sqlV4ToolFTS); err != nil { - return fmt.Errorf("create tool_fts: %w", err) - } - if _, err := tx.Exec(sqlV4Triggers); err != nil { - return fmt.Errorf("rebuild triggers: %w", err) - } - if _, err := tx.Exec(sqlV4Repopulate); err != nil { - return fmt.Errorf("repopulate indexes: %w", err) - } - - checksum := sha256.Sum256([]byte(sqlV4ToolFTS + sqlV4Triggers)) - checksumHex := fmt.Sprintf("%x", checksum) - - _, err = tx.Exec(` - INSERT INTO schema_migrations (version, name, applied_on, checksum) - VALUES (4, 'V4 tool_fts trigram index', CURRENT_TIMESTAMP, ?) - `, checksumHex) - if err != nil { - return fmt.Errorf("record migration v4: %w", err) - } - - return tx.Commit() + return d.applySingleMigration(applyV4) } // applyV5Migration drops the phantom session_events table. Nothing reads or @@ -328,32 +207,7 @@ func (d *Database) applyV4Migration() error { // Per the schema rule this is a new migration; V1 still creates the table on // the way up, and V5 drops it. func (d *Database) applyV5Migration() error { - tx, err := d.db.Begin() - if err != nil { - return fmt.Errorf("begin transaction: %w", err) - } - defer func() { _ = tx.Rollback() }() - - const sqlV5Drop = ` -DROP INDEX IF EXISTS idx_session_events_order; -DROP INDEX IF EXISTS idx_session_events_project; -DROP TABLE IF EXISTS session_events; -` - if _, err := tx.Exec(sqlV5Drop); err != nil { - return fmt.Errorf("drop session_events: %w", err) - } - - checksum := sha256.Sum256([]byte(sqlV5Drop)) - checksumHex := fmt.Sprintf("%x", checksum) - - if _, err := tx.Exec(` - INSERT INTO schema_migrations (version, name, applied_on, checksum) - VALUES (5, 'V5 drop phantom session_events', CURRENT_TIMESTAMP, ?) - `, checksumHex); err != nil { - return fmt.Errorf("record migration v5: %w", err) - } - - return tx.Commit() + return d.applySingleMigration(applyV5) } // applyV6Migration drops the phantom source_metadata column. Nothing reads or @@ -361,29 +215,7 @@ DROP TABLE IF EXISTS session_events; // Per the schema rule this is a new migration; V1 still creates the column on // the way up, and V6 drops it. func (d *Database) applyV6Migration() error { - tx, err := d.db.Begin() - if err != nil { - return fmt.Errorf("begin transaction: %w", err) - } - defer func() { _ = tx.Rollback() }() - - const sqlV6Drop = `ALTER TABLE search_items DROP COLUMN source_metadata;` - - if _, err := tx.Exec(sqlV6Drop); err != nil { - return fmt.Errorf("drop source_metadata column: %w", err) - } - - checksum := sha256.Sum256([]byte(sqlV6Drop)) - checksumHex := fmt.Sprintf("%x", checksum) - - if _, err := tx.Exec(` - INSERT INTO schema_migrations (version, name, applied_on, checksum) - VALUES (6, 'V6 drop phantom source_metadata column', CURRENT_TIMESTAMP, ?) - `, checksumHex); err != nil { - return fmt.Errorf("record migration v6: %w", err) - } - - return tx.Commit() + return d.applySingleMigration(applyV6) } // applyV7Migration updates the content_type-branched triggers to support reasoning @@ -392,28 +224,7 @@ func (d *Database) applyV6Migration() error { // tool_fts is for structured tool metadata (names, paths, commands); messages_fts // is for prose (text, code, reasoning). func (d *Database) applyV7Migration() error { - tx, err := d.db.Begin() - if err != nil { - return fmt.Errorf("begin transaction: %w", err) - } - defer func() { _ = tx.Rollback() }() - - if _, err := tx.Exec(sqlV7Triggers); err != nil { - return fmt.Errorf("rebuild triggers for reasoning: %w", err) - } - - checksum := sha256.Sum256([]byte(sqlV7Triggers)) - checksumHex := fmt.Sprintf("%x", checksum) - - _, err = tx.Exec(` - INSERT INTO schema_migrations (version, name, applied_on, checksum) - VALUES (7, 'V7 reasoning content_type routes to messages_fts', CURRENT_TIMESTAMP, ?) - `, checksumHex) - if err != nil { - return fmt.Errorf("record migration v7: %w", err) - } - - return tx.Commit() + return d.applySingleMigration(applyV7) } // SQL schema strings @@ -707,79 +518,11 @@ CREATE TABLE IF NOT EXISTS dynamic_stopwords (term TEXT PRIMARY KEY); // NOT re-derivable once source files expire — no CASCADE lifecycle; only // purge deletes from it, explicitly. func (d *Database) applyV8Migration() error { - tx, err := d.db.Begin() - if err != nil { - return fmt.Errorf("begin transaction: %w", err) - } - defer func() { _ = tx.Rollback() }() - - const sqlV8 = ` -ALTER TABLE search_items ADD COLUMN extraction_version INTEGER; -ALTER TABLE search_items ADD COLUMN was_interrupted INTEGER; -CREATE TABLE IF NOT EXISTS tool_events ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - message_uuid TEXT, - source_path TEXT NOT NULL, - ordinal INTEGER NOT NULL, - tool_name TEXT NOT NULL, - command_head TEXT, - is_error INTEGER, - exit_code INTEGER, - extraction_version INTEGER NOT NULL, - UNIQUE(source_path, ordinal) -); -CREATE INDEX IF NOT EXISTS idx_tool_events_tool ON tool_events(tool_name); -CREATE INDEX IF NOT EXISTS idx_tool_events_uuid ON tool_events(message_uuid); -` - if _, err := tx.Exec(sqlV8); err != nil { - return fmt.Errorf("apply v8 perennity schema: %w", err) - } - - checksum := sha256.Sum256([]byte(sqlV8)) - checksumHex := fmt.Sprintf("%x", checksum) - - if _, err := tx.Exec(` - INSERT INTO schema_migrations (version, name, applied_on, checksum) - VALUES (8, 'V8 perennity: extraction_version, was_interrupted, tool_events', CURRENT_TIMESTAMP, ?) - `, checksumHex); err != nil { - return fmt.Errorf("record migration v8: %w", err) - } - - return tx.Commit() + return d.applySingleMigration(applyV8) } func (d *Database) applyV9Migration() error { - tx, err := d.db.Begin() - if err != nil { - return fmt.Errorf("begin transaction: %w", err) - } - defer func() { _ = tx.Rollback() }() - - // Dedupe before indexing: v8 code could accumulate the same message_uuid - // at different ordinals (ordinal drift) or across files; creating the - // unique index on such data would fail on every startup. Keep the oldest - // row per uuid. - const sqlV9 = ` -DELETE FROM tool_events WHERE message_uuid IS NOT NULL AND id NOT IN ( - SELECT MIN(id) FROM tool_events WHERE message_uuid IS NOT NULL GROUP BY message_uuid -); -CREATE UNIQUE INDEX IF NOT EXISTS idx_tool_events_uuid_unique ON tool_events(message_uuid) WHERE message_uuid IS NOT NULL; -` - if _, err := tx.Exec(sqlV9); err != nil { - return fmt.Errorf("apply v9 tool_events uuid uniqueness: %w", err) - } - - checksum := sha256.Sum256([]byte(sqlV9)) - checksumHex := fmt.Sprintf("%x", checksum) - - if _, err := tx.Exec(` - INSERT INTO schema_migrations (version, name, applied_on, checksum) - VALUES (9, 'V9 tool_events uuid uniqueness index', CURRENT_TIMESTAMP, ?) - `, checksumHex); err != nil { - return fmt.Errorf("record migration v9: %w", err) - } - - return tx.Commit() + return d.applySingleMigration(applyV9) } // applyV10Migration adds F2 template mining surface: message_templates @@ -788,52 +531,7 @@ CREATE UNIQUE INDEX IF NOT EXISTS idx_tool_events_uuid_unique ON tool_events(mes // idempotent under re-sync). Only templates with occurrence_count >= 3 // (configurable) are reported; mining runs inside SyncFiles tx. func (d *Database) applyV10Migration() error { - tx, err := d.db.Begin() - if err != nil { - return fmt.Errorf("begin transaction: %w", err) - } - defer func() { _ = tx.Rollback() }() - - const sqlV10 = ` -CREATE TABLE IF NOT EXISTS message_templates ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - signature TEXT UNIQUE NOT NULL, - normalization_version INTEGER NOT NULL, - template_text TEXT NOT NULL, - occurrence_count INTEGER NOT NULL DEFAULT 1, - first_seen TEXT, - last_seen TEXT -); -CREATE INDEX IF NOT EXISTS idx_templates_sig ON message_templates(signature); -CREATE INDEX IF NOT EXISTS idx_templates_version ON message_templates(normalization_version); - -CREATE TABLE IF NOT EXISTS template_matches ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - template_id INTEGER NOT NULL, - item_uuid TEXT, - source_path TEXT NOT NULL, - ordinal INTEGER NOT NULL, - UNIQUE(source_path, ordinal, template_id), - FOREIGN KEY(template_id) REFERENCES message_templates(id) -); -CREATE INDEX IF NOT EXISTS idx_matches_template ON template_matches(template_id); -CREATE INDEX IF NOT EXISTS idx_matches_uuid ON template_matches(item_uuid); -` - if _, err := tx.Exec(sqlV10); err != nil { - return fmt.Errorf("apply v10 template-mining schema: %w", err) - } - - checksum := sha256.Sum256([]byte(sqlV10)) - checksumHex := fmt.Sprintf("%x", checksum) - - if _, err := tx.Exec(` - INSERT INTO schema_migrations (version, name, applied_on, checksum) - VALUES (10, 'V10 template mining: message_templates, template_matches', CURRENT_TIMESTAMP, ?) - `, checksumHex); err != nil { - return fmt.Errorf("record migration v10: %w", err) - } - - return tx.Commit() + return d.applySingleMigration(applyV10) } // applyV11Migration adds the F3 correction-detection surface: the perennial @@ -841,112 +539,19 @@ CREATE INDEX IF NOT EXISTS idx_matches_uuid ON template_matches(item_uuid); // by message identity). One row per (source_path, ordinal, detector) tuple. // extraction_version tracks detector evolution (like message extraction_version). func (d *Database) applyV11Migration() error { - tx, err := d.db.Begin() - if err != nil { - return fmt.Errorf("begin transaction: %w", err) - } - defer func() { _ = tx.Rollback() }() - - const sqlV11 = ` -CREATE TABLE IF NOT EXISTS correction_signals ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - item_uuid TEXT, - source_path TEXT NOT NULL, - ordinal INTEGER NOT NULL, - detector TEXT NOT NULL, - confidence REAL NOT NULL, - extraction_version INTEGER NOT NULL, - UNIQUE(source_path, ordinal, detector) -); -CREATE INDEX IF NOT EXISTS idx_correction_signals_detector ON correction_signals(detector); -CREATE INDEX IF NOT EXISTS idx_correction_signals_confidence ON correction_signals(confidence DESC); -` - if _, err := tx.Exec(sqlV11); err != nil { - return fmt.Errorf("apply v11 correction_signals schema: %w", err) - } - - checksum := sha256.Sum256([]byte(sqlV11)) - checksumHex := fmt.Sprintf("%x", checksum) - - if _, err := tx.Exec(` - INSERT INTO schema_migrations (version, name, applied_on, checksum) - VALUES (11, 'V11 correction detection: correction_signals', CURRENT_TIMESTAMP, ?) - `, checksumHex); err != nil { - return fmt.Errorf("record migration v11: %w", err) - } - - return tx.Commit() + return d.applySingleMigration(applyV11) } // applyV12Migration adds the F3b agent-classification surface: the perennial // annotations table (one row per message per kind; re-annotating replaces). // Labels are free-form in v1; label_enum freezing is a future slice (post-calibration). func (d *Database) applyV12Migration() error { - tx, err := d.db.Begin() - if err != nil { - return fmt.Errorf("begin transaction: %w", err) - } - defer func() { _ = tx.Rollback() }() - - const sqlV12 = ` -CREATE TABLE IF NOT EXISTS annotations ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - item_uuid TEXT, - source_path TEXT NOT NULL, - ordinal INTEGER NOT NULL, - kind TEXT NOT NULL, - label TEXT NOT NULL, - source TEXT NOT NULL DEFAULT 'agent', - created_at TEXT NOT NULL, - UNIQUE(source_path, ordinal, kind) -); -CREATE INDEX IF NOT EXISTS idx_annotations_uuid ON annotations(item_uuid); -CREATE INDEX IF NOT EXISTS idx_annotations_kind ON annotations(kind); -` - if _, err := tx.Exec(sqlV12); err != nil { - return fmt.Errorf("apply v12 annotations schema: %w", err) - } - - checksum := sha256.Sum256([]byte(sqlV12)) - checksumHex := fmt.Sprintf("%x", checksum) - - if _, err := tx.Exec(` - INSERT INTO schema_migrations (version, name, applied_on, checksum) - VALUES (12, 'V12 agent classification: annotations (free-form labels; enum freeze deferred)', CURRENT_TIMESTAMP, ?) - `, checksumHex); err != nil { - return fmt.Errorf("record migration v12: %w", err) - } - - return tx.Commit() + return d.applySingleMigration(applyV12) } // applyV13Migration adds indexes on template_matches and correction_signals // for efficient backfill discovery queries. The queries use NOT EXISTS subqueries // on source_path; indexes reduce from O(N·M) table scans to O(N·log M) index lookups. func (d *Database) applyV13Migration() error { - tx, err := d.db.Begin() - if err != nil { - return fmt.Errorf("begin transaction: %w", err) - } - defer func() { _ = tx.Rollback() }() - - const sqlV13 = ` -CREATE INDEX IF NOT EXISTS idx_template_matches_source ON template_matches(source_path); -CREATE INDEX IF NOT EXISTS idx_correction_signals_source ON correction_signals(source_path); -` - if _, err := tx.Exec(sqlV13); err != nil { - return fmt.Errorf("apply v13 indexes: %w", err) - } - - checksum := sha256.Sum256([]byte(sqlV13)) - checksumHex := fmt.Sprintf("%x", checksum) - - if _, err := tx.Exec(` - INSERT INTO schema_migrations (version, name, applied_on, checksum) - VALUES (13, 'V13 backfill discovery indexes', CURRENT_TIMESTAMP, ?) - `, checksumHex); err != nil { - return fmt.Errorf("record migration v13: %w", err) - } - - return tx.Commit() + return d.applySingleMigration(applyV13) } diff --git a/internal/storage/records.go b/internal/storage/records.go index 6f20d7a..a2d4ef8 100644 --- a/internal/storage/records.go +++ b/internal/storage/records.go @@ -4,20 +4,9 @@ import ( "database/sql" "fmt" "strings" -) -// IndexedRecord represents a single record from search_items. -type IndexedRecord struct { - Source string - SourcePath string - Ordinal int64 - Role string - Text string - Project *string - UUID *string - Timestamp *string - ContentType string -} + "github.com/pablontiv/backscroll/internal/models" +) // IndexedRecordQuery defines filter parameters for QueryIndexedRecords. type IndexedRecordQuery struct { @@ -32,7 +21,7 @@ type IndexedRecordQuery struct { // QueryIndexedRecords returns records from search_items matching the query, // ordered by source_path and ordinal. -func (d *Database) QueryIndexedRecords(q IndexedRecordQuery) ([]IndexedRecord, error) { +func (d *Database) QueryIndexedRecords(q IndexedRecordQuery) ([]models.IndexedRecord, error) { baseQuery := ` SELECT source, source_path, ordinal, role, text, project, uuid, timestamp, content_type FROM search_items` @@ -80,9 +69,9 @@ func (d *Database) QueryIndexedRecords(q IndexedRecordQuery) ([]IndexedRecord, e } defer func() { _ = rows.Close() }() - var records []IndexedRecord + var records []models.IndexedRecord for rows.Next() { - var r IndexedRecord + var r models.IndexedRecord var project, uuid, timestamp sql.NullString if err := rows.Scan( &r.Source, &r.SourcePath, &r.Ordinal, &r.Role, &r.Text, diff --git a/internal/storage/recovery_destination.go b/internal/storage/recovery_destination.go new file mode 100644 index 0000000..f7ba976 --- /dev/null +++ b/internal/storage/recovery_destination.go @@ -0,0 +1,521 @@ +package storage + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "os" + "regexp" + "sort" + "strconv" + "strings" + + "github.com/google/uuid" + "github.com/pablontiv/backscroll/internal/compat" + "github.com/pablontiv/backscroll/internal/models" +) + +var recoveryFTSTokenRE = regexp.MustCompile(`[A-Za-z0-9_]{3,}`) + +// RecoveryDestinationError describes a failed destination construction attempt. +// Path is populated when a temporary destination path may have leaked; CleanupErr +// is populated when cleanup of that exact path failed. +type RecoveryDestinationError struct { + Path string + Cause error + CleanupErr error +} + +func (e *RecoveryDestinationError) Error() string { + if e.Path != "" { + return fmt.Sprintf("recovery destination %s failed: %v", e.Path, e.Cause) + } + return fmt.Sprintf("recovery destination failed: %v", e.Cause) +} + +func (e *RecoveryDestinationError) Unwrap() []error { + var errs []error + if e.Cause != nil { + errs = append(errs, e.Cause) + } + if e.CleanupErr != nil { + errs = append(errs, e.CleanupErr) + } + return errs +} + +func recoveryDestinationError(path string, cause, cleanupErr error) error { + if cause == nil { + cause = cleanupErr + } + return &RecoveryDestinationError{Path: path, Cause: cause, CleanupErr: cleanupErr} +} + +// CreateRecoveryDestination creates a fresh current-schema recovery database in +// dir, imports the applicable canonical plan in one transaction, verifies the +// uncommitted transaction, commits it, then independently reopens the committed +// bytes read-only and verifies them again. The returned path is safe to expose +// only after that independent verification succeeds. +func CreateRecoveryDestination(ctx context.Context, dir string, plan compat.RecoveryPlan) (path string, err error) { + if plan.Records == nil { + return "", fmt.Errorf("recovery plan is inapplicable or contains diagnostics") + } + + temp, err := os.CreateTemp(dir, ".backscroll-recover-*.db") + if err != nil { + return "", fmt.Errorf("create recovery destination temp: %w", err) + } + path = temp.Name() + if closeErr := temp.Close(); closeErr != nil { + cause := fmt.Errorf("close recovery destination temp: %w", closeErr) + cleanupErr := recoveryDestinationRemoveFiles(path) + if cleanupErr != nil { + return path, recoveryDestinationError(path, cause, fmt.Errorf("cleanup leaked recovery destination temp %s: %w", path, cleanupErr)) + } + return "", cause + } + if err := os.Chmod(path, 0o600); err != nil { + cause := fmt.Errorf("set recovery destination permissions: %w", err) + cleanupErr := recoveryDestinationRemoveFiles(path) + if cleanupErr != nil { + return path, recoveryDestinationError(path, cause, fmt.Errorf("cleanup leaked recovery destination temp %s: %w", path, cleanupErr)) + } + return "", cause + } + + tempPath := path + cleanup := true + defer func() { + if cleanup { + if cleanupErr := recoveryDestinationRemoveFiles(tempPath); cleanupErr != nil { + path = tempPath + err = recoveryDestinationError(tempPath, err, fmt.Errorf("cleanup leaked recovery destination temp %s: %w", tempPath, cleanupErr)) + } else { + path = "" + } + } + }() + + db, err := Open(path) + if err != nil { + return "", fmt.Errorf("initialize fresh recovery destination: %w", err) + } + closeDB := true + defer func() { + if closeDB { + if closeErr := db.Close(); closeErr != nil { + err = errors.Join(err, fmt.Errorf("close recovery destination database: %w", closeErr)) + } + } + }() + + tx, err := db.DB().BeginTx(ctx, nil) + if err != nil { + return "", fmt.Errorf("begin recovery import transaction: %w", err) + } + committed := false + defer func() { + if !committed { + if rollbackErr := tx.Rollback(); rollbackErr != nil && !errors.Is(rollbackErr, sql.ErrTxDone) { + err = errors.Join(err, fmt.Errorf("rollback recovery import transaction: %w", rollbackErr)) + } + } + }() + + for i, planned := range plan.Records { + if err := insertRecoveryDestinationRecord(ctx, tx, planned.Record); err != nil { + return "", fmt.Errorf("insert recovery record %d: %w", i, err) + } + } + if err := verifyRecoveryDestinationQueryer(ctx, tx, plan); err != nil { + return "", fmt.Errorf("verify recovery destination before commit: %w", err) + } + if err := tx.Commit(); err != nil { + return "", fmt.Errorf("commit recovery import transaction: %w", err) + } + committed = true + if _, err := db.DB().ExecContext(ctx, `PRAGMA wal_checkpoint(TRUNCATE)`); err != nil { + return "", fmt.Errorf("checkpoint recovery destination into main database: %w", err) + } + + if err := db.Close(); err != nil { + closeDB = false + return "", fmt.Errorf("close recovery destination before independent verification: %w", err) + } + closeDB = false + if err := removeEmptyRecoveryDestinationSidecars(path); err != nil { + return "", err + } + + if err := VerifyRecoveryDestination(ctx, path, plan); err != nil { + return "", fmt.Errorf("verify committed recovery destination: %w", err) + } + cleanup = false + return path, nil +} + +// VerifyRecoveryDestination independently opens path read-only and verifies that +// its committed bytes exactly match the current-schema canonical recovery plan. +func removeEmptyRecoveryDestinationSidecars(path string) error { + var errs []error + for _, suffix := range []string{"-wal", "-shm", "-journal"} { + sidecar := path + suffix + info, err := os.Lstat(sidecar) + if os.IsNotExist(err) { + continue + } + if err != nil { + errs = append(errs, fmt.Errorf("lstat recovery destination sidecar %s: %w", sidecar, err)) + continue + } + if !info.Mode().IsRegular() || info.Size() != 0 { + errs = append(errs, fmt.Errorf("recovery destination has unexpected SQLite sidecar %s", sidecar)) + continue + } + if err := os.Remove(sidecar); err != nil && !os.IsNotExist(err) { + errs = append(errs, fmt.Errorf("remove empty recovery destination sidecar %s: %w", sidecar, err)) + } + } + return errors.Join(errs...) +} + +func VerifyRecoveryDestination(ctx context.Context, path string, plan compat.RecoveryPlan) (err error) { + if plan.Records == nil { + return fmt.Errorf("recovery plan is inapplicable or contains diagnostics") + } + db, err := OpenImmutableReadOnly(path) + if err != nil { + return fmt.Errorf("open recovery destination immutable read-only: %w", err) + } + defer func() { + if closeErr := db.Close(); closeErr != nil { + err = errors.Join(err, fmt.Errorf("close recovery destination immutable read-only %s: %w", path, closeErr)) + } + }() + return verifyRecoveryDestinationQueryer(ctx, db.DB(), plan) +} + +func insertRecoveryDestinationRecord(ctx context.Context, tx *sql.Tx, r models.IndexedRecord) error { + // extraction_version and was_interrupted are intentionally NULL here: the + // canonical recovery record does not carry that lossy derived metadata, and + // recovered rows remain eligible for safe rederivation by future sync/mining. + _, err := tx.ExecContext(ctx, ` + INSERT INTO search_items (source, source_path, ordinal, role, text, timestamp, uuid, project, content_type, extraction_version, was_interrupted) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL) + `, r.Source, r.SourcePath, r.Ordinal, r.Role, r.Text, recoveryDestinationNullableString(r.Timestamp), recoveryDestinationUUIDValue(r.UUID), recoveryDestinationNullableString(r.Project), r.ContentType) + if err != nil { + return fmt.Errorf("insert search_items: %w", err) + } + return nil +} + +func verifyRecoveryDestinationQueryer(ctx context.Context, q compat.Queryer, plan compat.RecoveryPlan) error { + if err := compat.VerifyCurrentShape(ctx, q); err != nil { + return fmt.Errorf("current schema signature: %w", err) + } + if err := verifyRecoveryDestinationForeignKeys(ctx, q); err != nil { + return err + } + if err := verifyRecoveryDestinationRows(ctx, q, plan); err != nil { + return err + } + if err := verifyRecoveryDestinationFTS(ctx, q, plan); err != nil { + return err + } + return nil +} + +func verifyRecoveryDestinationRows(ctx context.Context, q compat.Queryer, plan compat.RecoveryPlan) error { + var rowCount int + if err := q.QueryRowContext(ctx, `SELECT COUNT(*) FROM search_items`).Scan(&rowCount); err != nil { + return fmt.Errorf("count search_items: %w", err) + } + if rowCount != len(plan.Records) { + return fmt.Errorf("search_items count = %d, want %d", rowCount, len(plan.Records)) + } + var indexedFileCount int + if err := q.QueryRowContext(ctx, `SELECT COUNT(*) FROM indexed_files`).Scan(&indexedFileCount); err != nil { + return fmt.Errorf("count indexed_files: %w", err) + } + if indexedFileCount != 0 { + return fmt.Errorf("indexed_files count = %d, want 0 for recovery destination source accounting", indexedFileCount) + } + + records, diag, err := readCanonicalSearchItems(ctx, q) + if err != nil { + return err + } + if diag != nil { + return fmt.Errorf("%s: %s", diag.Code, diag.Summary) + } + if len(records) != len(plan.Records) { + return fmt.Errorf("canonical record count = %d, want %d", len(records), len(plan.Records)) + } + + wantIdentities := map[string]bool{} + wantPayloads := map[string]bool{} + wantSources := map[string]int{} + for i, planned := range plan.Records { + identity, err := recoveryDestinationIdentity(planned.Record) + if err != nil { + return fmt.Errorf("planned record %d identity: %w", i, err) + } + if wantIdentities[identity] { + return fmt.Errorf("planned record %d repeats identity %s", i, identity) + } + wantIdentities[identity] = true + payload, err := recoveryDestinationPayload(planned.Record) + if err != nil { + return err + } + wantPayloads[identity+"\x00"+payload] = true + wantSources[planned.Record.Source]++ + } + + gotIdentities := map[string]bool{} + gotPayloads := map[string]bool{} + gotSources := map[string]int{} + for i, record := range records { + identity, err := recoveryDestinationIdentity(record) + if err != nil { + return fmt.Errorf("destination record %d identity: %w", i, err) + } + if gotIdentities[identity] { + return fmt.Errorf("destination repeats identity %s", identity) + } + gotIdentities[identity] = true + payload, err := recoveryDestinationPayload(record) + if err != nil { + return err + } + gotPayloads[identity+"\x00"+payload] = true + gotSources[record.Source]++ + } + if !recoveryDestinationStringBoolMapsEqual(gotIdentities, wantIdentities) { + return fmt.Errorf("destination identities do not match plan") + } + if !recoveryDestinationStringBoolMapsEqual(gotPayloads, wantPayloads) { + return fmt.Errorf("destination payloads do not match plan") + } + if !recoveryDestinationStringIntMapsEqual(gotSources, wantSources) { + return fmt.Errorf("destination source accounting does not match plan") + } + return nil +} + +func verifyRecoveryDestinationForeignKeys(ctx context.Context, q compat.Queryer) (err error) { + rows, err := q.QueryContext(ctx, `PRAGMA foreign_key_check`) + if err != nil { + return fmt.Errorf("foreign_key_check: %w", err) + } + defer func() { + if closeErr := rows.Close(); closeErr != nil { + err = errors.Join(err, fmt.Errorf("close foreign_key_check rows: %w", closeErr)) + } + }() + if rows.Next() { + return fmt.Errorf("foreign_key_check reported violations") + } + if err := rows.Err(); err != nil { + return fmt.Errorf("read foreign_key_check: %w", err) + } + return nil +} + +func verifyRecoveryDestinationFTS(ctx context.Context, q compat.Queryer, plan compat.RecoveryPlan) error { + checkedMessages, checkedTools := 0, 0 + for i, planned := range plan.Records { + record := planned.Record + switch record.ContentType { + case "tool": + checked, err := verifyRecoveryDestinationFTSRecord(ctx, q, "tool_fts", "messages_fts", record) + if err != nil { + return fmt.Errorf("verify tool_fts record %d: %w", i, err) + } + if checked { + checkedTools++ + } + case "text", "code", "reasoning": + checked, err := verifyRecoveryDestinationFTSRecord(ctx, q, "messages_fts", "tool_fts", record) + if err != nil { + return fmt.Errorf("verify messages_fts record %d: %w", i, err) + } + if checked { + checkedMessages++ + } + } + } + if want := recoveryDestinationTokenedCount(plan, "messages_fts"); checkedMessages != want { + return fmt.Errorf("messages_fts verified count = %d, want %d", checkedMessages, want) + } + if want := recoveryDestinationTokenedCount(plan, "tool_fts"); checkedTools != want { + return fmt.Errorf("tool_fts verified count = %d, want %d", checkedTools, want) + } + return nil +} + +func verifyRecoveryDestinationFTSRecord(ctx context.Context, q compat.Queryer, table, wrongTable string, record models.IndexedRecord) (bool, error) { + match := recoveryDestinationFTSToken(record.Text) + if match == "" { + return false, nil + } + id, err := recoveryDestinationRecordID(ctx, q, record) + if err != nil { + return false, fmt.Errorf("locate %s row: %w", table, err) + } + if err := verifyRecoveryDestinationFTSCount(ctx, q, table, match, id, 1); err != nil { + return false, err + } + if err := verifyRecoveryDestinationFTSCount(ctx, q, wrongTable, match, id, 0); err != nil { + return false, err + } + return true, nil +} + +func verifyRecoveryDestinationFTSCount(ctx context.Context, q compat.Queryer, table, match string, id int64, want int) error { + query, err := recoveryDestinationFTSCountSQL(table) + if err != nil { + return err + } + var got int + if err := q.QueryRowContext(ctx, query, match, id).Scan(&got); err != nil { + return fmt.Errorf("count %s virtual table hits for rowid %d: %w", table, id, err) + } + if got != want { + return fmt.Errorf("%s virtual table hits for rowid %d token %q = %d, want %d", table, id, match, got, want) + } + return nil +} + +func recoveryDestinationFTSCountSQL(table string) (string, error) { + switch table { + case "messages_fts": + return `SELECT COUNT(*) FROM messages_fts WHERE messages_fts MATCH ? AND rowid = ?`, nil + case "tool_fts": + return `SELECT COUNT(*) FROM tool_fts WHERE tool_fts MATCH ? AND rowid = ?`, nil + default: + return "", fmt.Errorf("unknown FTS table %q", table) + } +} + +func recoveryDestinationRecordID(ctx context.Context, q compat.Queryer, record models.IndexedRecord) (int64, error) { + var id int64 + if err := q.QueryRowContext(ctx, ` + SELECT id FROM search_items + WHERE source_path = ? AND ordinal = ? AND COALESCE(uuid, '') = ? AND text = ? + `, record.SourcePath, record.Ordinal, recoveryDestinationStringValue(record.UUID), record.Text).Scan(&id); err != nil { + return 0, err + } + return id, nil +} + +func recoveryDestinationTokenedCount(plan compat.RecoveryPlan, table string) int { + count := 0 + for _, planned := range plan.Records { + record := planned.Record + if recoveryDestinationFTSToken(record.Text) == "" { + continue + } + if table == "tool_fts" && record.ContentType == "tool" { + count++ + } + if table == "messages_fts" && (record.ContentType == "text" || record.ContentType == "code" || record.ContentType == "reasoning") { + count++ + } + } + return count +} + +func recoveryDestinationIdentity(r models.IndexedRecord) (string, error) { + if r.UUID != nil && *r.UUID != "" { + if _, err := uuid.Parse(*r.UUID); err != nil { + return "", err + } + return "uuid\x00" + *r.UUID, nil + } + if r.SourcePath == "" || r.Ordinal < 0 { + return "", fmt.Errorf("unsafe path/ordinal identity source_path=%q ordinal=%d", r.SourcePath, r.Ordinal) + } + return "path_ordinal\x00" + r.SourcePath + "\x00" + strconv.FormatInt(r.Ordinal, 10), nil +} + +func recoveryDestinationPayload(r models.IndexedRecord) (string, error) { + if r.UUID != nil && *r.UUID == "" { + r.UUID = nil + } + encoded, err := json.Marshal(r) + if err != nil { + return "", fmt.Errorf("encode recovery payload: %w", err) + } + return string(encoded), nil +} + +func recoveryDestinationNullableString(value *string) any { + if value == nil { + return nil + } + return *value +} + +func recoveryDestinationUUIDValue(value *string) any { + if value == nil || *value == "" { + return nil + } + return *value +} + +func recoveryDestinationStringValue(value *string) string { + if value == nil { + return "" + } + return *value +} + +func recoveryDestinationFTSToken(text string) string { + tokens := recoveryFTSTokenRE.FindAllString(text, -1) + if len(tokens) == 0 { + return "" + } + sort.SliceStable(tokens, func(i, j int) bool { return len(tokens[i]) > len(tokens[j]) }) + return strings.ToLower(tokens[0]) +} + +func recoveryDestinationStringBoolMapsEqual(left, right map[string]bool) bool { + if len(left) != len(right) { + return false + } + for key, leftValue := range left { + if right[key] != leftValue { + return false + } + } + return true +} + +func recoveryDestinationStringIntMapsEqual(left, right map[string]int) bool { + if len(left) != len(right) { + return false + } + for key, leftValue := range left { + if right[key] != leftValue { + return false + } + } + return true +} + +func recoveryDestinationRemoveFiles(path string) error { + if path == "" { + return nil + } + var errs []error + for _, suffix := range []string{"", "-wal", "-shm", "-journal"} { + candidate := path + suffix + if err := os.Remove(candidate); err != nil && !os.IsNotExist(err) { + errs = append(errs, fmt.Errorf("remove recovery destination temp %s: %w", candidate, err)) + } + } + return errors.Join(errs...) +} diff --git a/internal/storage/recovery_destination_test.go b/internal/storage/recovery_destination_test.go new file mode 100644 index 0000000..ea1a52d --- /dev/null +++ b/internal/storage/recovery_destination_test.go @@ -0,0 +1,811 @@ +package storage + +import ( + "context" + "database/sql" + "errors" + "os" + "path/filepath" + "reflect" + "sort" + "strconv" + "strings" + "testing" + "time" + + "github.com/pablontiv/backscroll/internal/compat" + "github.com/pablontiv/backscroll/internal/models" +) + +func TestPublishedGoLineagesUpgradeLosslessly(t *testing.T) { + ctx := context.Background() + catalog, err := compat.LoadCatalog() + if err != nil { + t.Fatal(err) + } + + seenFixtures := map[string][]string{} + for _, release := range catalog.Releases { + seenFixtures[release.Fixture] = append(seenFixtures[release.Fixture], release.Tag) + } + if len(seenFixtures) == 0 { + t.Fatal("catalog has no published Go releases") + } + + fixtures := make([]string, 0, len(seenFixtures)) + for fixture := range seenFixtures { + fixtures = append(fixtures, fixture) + } + sort.Strings(fixtures) + + for _, fixture := range fixtures { + fixture := fixture + t.Run(fixture, func(t *testing.T) { + dbPath := createFixtureDatabase(t, fixture) + want := seedPublishedReleaseRecoverySentinels(t, dbPath, fixture) + + db, diag, err := OpenCompatible(ctx, dbPath) + if err != nil || diag != nil { + t.Fatalf("OpenCompatible for published releases %v fixture %s err=%v diagnostic=%+v", seenFixtures[fixture], fixture, err, diag) + } + defer func() { _ = db.Close() }() + + assertCurrentShape(t, db.DB()) + assertPublishedReleaseSentinels(t, db.DB(), want) + assertFTSQueryable(t, db.DB(), "publishedlineagealpha", 1) + assertToolFTSQueryable(t, db.DB(), "publishedlineagecmd", 1) + + input, diag, err := ReadRecoveryInput(ctx, db) + if err != nil || diag != nil { + t.Fatalf("ReadRecoveryInput after published-release migration err=%v diagnostic=%+v", err, diag) + } + assertPublishedReleaseRecoveryInput(t, input, want) + plan, diagnostics, err := compat.PlanRecovery([]compat.RecoveryInput{input}) + if err != nil || len(diagnostics) != 0 { + t.Fatalf("PlanRecovery after published-release migration err=%v diagnostics=%+v", err, diagnostics) + } + if len(plan.Records) != len(want.Records) { + t.Fatalf("recovery plan records = %d, want %d", len(plan.Records), len(want.Records)) + } + + destPath, err := CreateRecoveryDestination(ctx, filepath.Dir(dbPath), plan) + if err != nil { + t.Fatalf("CreateRecoveryDestination from migrated published release: %v", err) + } + if err := VerifyRecoveryDestination(ctx, destPath, plan); err != nil { + t.Fatalf("VerifyRecoveryDestination from migrated published release: %v", err) + } + assertRecoveryDestinationRecords(t, destPath, plan) + assertRecoveryDestinationFTS(t, destPath, "publishedlineagealpha", 1, "publishedlineagecmd", 1) + }) + } +} + +type publishedReleaseSentinels struct { + Records []models.IndexedRecord + IndexedPath string + IndexedHash string +} + +func seedPublishedReleaseRecoverySentinels(t *testing.T, dbPath, fixture string) publishedReleaseSentinels { + t.Helper() + db, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatal(err) + } + defer func() { _ = db.Close() }() + + records := []models.IndexedRecord{ + { + Source: "session", + SourcePath: "/published/" + fixture + "/text.jsonl", + Ordinal: 0, + Role: "user", + Text: "publishedlineagealpha text survives migration", + Project: stringPtr("published-project"), + UUID: stringPtr("11111111-1111-4111-8111-111111111111"), + Timestamp: stringPtr("2026-08-18T00:00:00Z"), + ContentType: "text", + }, + { + Source: "session", + SourcePath: "/published/" + fixture + "/tool.jsonl", + Ordinal: 1, + Role: "assistant", + Text: "publishedlineagecmd tool survives migration", + Project: stringPtr("published-project"), + UUID: stringPtr("22222222-2222-4222-8222-222222222222"), + Timestamp: stringPtr("2026-08-18T00:00:01Z"), + ContentType: "tool", + }, + } + for _, record := range records { + if _, err := db.Exec(`INSERT INTO search_items (source, source_path, ordinal, role, text, timestamp, uuid, project, content_type) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, record.Source, record.SourcePath, record.Ordinal, record.Role, record.Text, pointerTestString(record.Timestamp), pointerTestString(record.UUID), pointerTestString(record.Project), record.ContentType); err != nil { + t.Fatalf("insert published release sentinel into %s: %v", fixture, err) + } + } + + indexedPath := "/published/" + fixture + "/indexed.jsonl" + indexedHash := "published-hash-" + fixture + if _, err := db.Exec(`INSERT INTO indexed_files (path, hash, last_indexed) VALUES (?, ?, ?)`, indexedPath, indexedHash, "2026-08-18T00:00:02Z"); err != nil { + t.Fatalf("insert published release indexed_files sentinel into %s: %v", fixture, err) + } + return publishedReleaseSentinels{Records: records, IndexedPath: indexedPath, IndexedHash: indexedHash} +} + +func assertPublishedReleaseSentinels(t *testing.T, db *sql.DB, want publishedReleaseSentinels) { + t.Helper() + for _, wantRecord := range want.Records { + var got models.IndexedRecord + var project, uuid, timestamp string + if err := db.QueryRow(`SELECT source, source_path, ordinal, role, text, project, uuid, timestamp, content_type + FROM search_items WHERE uuid = ?`, pointerTestString(wantRecord.UUID)).Scan(&got.Source, &got.SourcePath, &got.Ordinal, &got.Role, &got.Text, &project, &uuid, ×tamp, &got.ContentType); err != nil { + t.Fatalf("query migrated published release sentinel %s: %v", pointerTestString(wantRecord.UUID), err) + } + got.Project = &project + got.UUID = &uuid + got.Timestamp = ×tamp + if !reflect.DeepEqual(got, wantRecord) { + t.Fatalf("migrated published release sentinel mismatch\ngot: %+v\nwant: %+v", got, wantRecord) + } + } + var gotHash string + if err := db.QueryRow(`SELECT hash FROM indexed_files WHERE path = ?`, want.IndexedPath).Scan(&gotHash); err != nil { + t.Fatalf("query migrated published release indexed_files sentinel: %v", err) + } + if gotHash != want.IndexedHash { + t.Fatalf("indexed_files hash = %q, want %q", gotHash, want.IndexedHash) + } +} + +func assertPublishedReleaseRecoveryInput(t *testing.T, input compat.RecoveryInput, want publishedReleaseSentinels) { + t.Helper() + if input.RowCount != len(want.Records) || len(input.Records) != len(want.Records) { + t.Fatalf("recovery input row count = %d records = %d, want %d", input.RowCount, len(input.Records), len(want.Records)) + } + got := append([]models.IndexedRecord(nil), input.Records...) + wantRecords := append([]models.IndexedRecord(nil), want.Records...) + sortRecoveryDestinationRecords(got) + sortRecoveryDestinationRecords(wantRecords) + if !reflect.DeepEqual(got, wantRecords) { + t.Fatalf("recovery input records mismatch\ngot: %+v\nwant: %+v", got, wantRecords) + } +} + +func stringPtr(value string) *string { + return &value +} + +func TestRecoveryDestinationErrorPublicBehavior(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + dir := t.TempDir() + defer func() { _ = os.Chmod(dir, 0o755) }() + + probe := filepath.Join(dir, "cleanup-permission-probe") + if err := os.WriteFile(probe, []byte("probe"), 0o600); err != nil { + t.Fatalf("write cleanup probe: %v", err) + } + if err := os.Chmod(dir, 0o555); err != nil { + t.Fatalf("make cleanup probe directory read-only: %v", err) + } + probeRemoveErr := os.Remove(probe) + if chmodErr := os.Chmod(dir, 0o755); chmodErr != nil { + t.Fatalf("restore cleanup probe directory permissions: %v", chmodErr) + } + if probeRemoveErr == nil { + t.Fatal("test filesystem allows removing directory entries without directory write permission; cannot deterministically exercise public cleanup-error path") + } + if err := os.Remove(probe); err != nil && !os.IsNotExist(err) { + t.Fatalf("remove cleanup probe after permission restore: %v", err) + } + + plan := compat.RecoveryPlan{Records: make([]compat.CanonicalRecord, 0, 2048)} + for i := 0; i < cap(plan.Records); i++ { + uuid := "11111111-1111-4111-8111-" + strconv.FormatInt(int64(100000000000+i), 10) + plan.Records = append(plan.Records, compat.CanonicalRecord{Record: models.IndexedRecord{Source: "session", SourcePath: "/sessions/cancelled.jsonl", Ordinal: int64(i), Role: "user", Text: "cancelled destination row", UUID: &uuid, ContentType: "text"}}) + } + + seenTemp := make(chan string, 1) + watchDone := make(chan struct{}) + go func() { + defer close(watchDone) + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + matches, _ := filepath.Glob(filepath.Join(dir, ".backscroll-recover-*.db")) + for _, match := range matches { + base := filepath.Base(match) + if strings.HasSuffix(base, "-wal") || strings.HasSuffix(base, "-shm") || strings.HasSuffix(base, "-journal") { + continue + } + _ = os.Chmod(dir, 0o555) + cancel() + seenTemp <- match + return + } + time.Sleep(time.Millisecond) + } + cancel() + }() + + path, err := CreateRecoveryDestination(ctx, dir, plan) + <-watchDone + if err == nil { + t.Fatalf("CreateRecoveryDestination succeeded at %q; want public cleanup error", path) + } + var destinationErr *RecoveryDestinationError + if !errors.As(err, &destinationErr) { + t.Fatalf("error %T %[1]v, want *RecoveryDestinationError from public CreateRecoveryDestination", err) + } + if destinationErr.Path == "" || path != destinationErr.Path { + t.Fatalf("returned path=%q typed path=%q, want leaked temp path surfaced", path, destinationErr.Path) + } + if destinationErr.Cause == nil || destinationErr.CleanupErr == nil { + t.Fatalf("RecoveryDestinationError cause/cleanup = %v/%v, want both populated", destinationErr.Cause, destinationErr.CleanupErr) + } + if unwrapped := destinationErr.Unwrap(); len(unwrapped) != 2 || unwrapped[0] != destinationErr.Cause || unwrapped[1] != destinationErr.CleanupErr { + t.Fatalf("unwrap = %#v, want cause then cleanup", unwrapped) + } + select { + case temp := <-seenTemp: + if destinationErr.Path != temp { + t.Fatalf("typed path = %q, want observed temp %q", destinationErr.Path, temp) + } + default: + t.Fatal("CreateRecoveryDestination returned before temp watcher observed a public temp path") + } +} + +func TestRecoveryDestinationPublicValidationFailures(t *testing.T) { + ctx := context.Background() + if _, err := CreateRecoveryDestination(ctx, t.TempDir(), compat.RecoveryPlan{}); err == nil || !strings.Contains(err.Error(), "inapplicable") { + t.Fatalf("CreateRecoveryDestination nil plan error = %v, want inapplicable plan", err) + } + if err := VerifyRecoveryDestination(ctx, filepath.Join(t.TempDir(), "missing.db"), compat.RecoveryPlan{}); err == nil || !strings.Contains(err.Error(), "inapplicable") { + t.Fatalf("VerifyRecoveryDestination nil plan error = %v, want inapplicable plan", err) + } + + validEmptyPlan := compat.RecoveryPlan{Records: []compat.CanonicalRecord{}} + if _, err := CreateRecoveryDestination(ctx, filepath.Join(t.TempDir(), "missing"), validEmptyPlan); err == nil || !strings.Contains(err.Error(), "create recovery destination temp") { + t.Fatalf("CreateRecoveryDestination missing dir error = %v, want create temp failure", err) + } + if err := VerifyRecoveryDestination(ctx, filepath.Join(t.TempDir(), "missing.db"), validEmptyPlan); err == nil || !strings.Contains(err.Error(), "open recovery destination immutable read-only") { + t.Fatalf("VerifyRecoveryDestination missing file error = %v, want open immutable failure", err) + } +} + +func TestRecoverUnionPreservesActiveAndStrandedRecords(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + activePath := filepath.Join(dir, "active.db") + strandedPath := filepath.Join(dir, "stranded.db") + + duplicateUUID := "33333333-3333-4333-8333-333333333333" + activeUniqueUUID := "11111111-1111-4111-8111-111111111111" + strandedUniqueUUID := "22222222-2222-4222-8222-222222222222" + createRecoveryDestinationSourceDB(t, activePath, []IndexedFile{ + recoveryDestinationIndexedFile("/sessions/active.jsonl", "active-hash", []IndexedMessage{{ + Ordinal: 0, Role: "user", Text: "active prose sentinelalpha", UUID: activeUniqueUUID, + Timestamp: "2026-08-18T00:00:00Z", ContentType: "text", ExtractionVersion: CurrentExtractionVersion, + }}), + recoveryDestinationIndexedFile("/sessions/duplicate.jsonl", "duplicate-active-hash", []IndexedMessage{{ + Ordinal: 0, Role: "assistant", Text: "shared duplicate prose", UUID: duplicateUUID, + Timestamp: "2026-08-18T00:00:02Z", ContentType: "text", ExtractionVersion: CurrentExtractionVersion, + }}), + }) + createRecoveryDestinationSourceDB(t, strandedPath, []IndexedFile{ + recoveryDestinationIndexedFile("/sessions/stranded.jsonl", "stranded-hash", []IndexedMessage{{ + Ordinal: 0, Role: "assistant", Text: "Bash command=strandedtoolomega", UUID: strandedUniqueUUID, + Timestamp: "2026-08-18T00:00:01Z", ContentType: "tool", ToolName: "Bash", CommandHead: "strandedtoolomega", ExtractionVersion: CurrentExtractionVersion, + }}), + recoveryDestinationIndexedFile("/sessions/duplicate.jsonl", "duplicate-stranded-hash", []IndexedMessage{{ + Ordinal: 0, Role: "assistant", Text: "shared duplicate prose", UUID: duplicateUUID, + Timestamp: "2026-08-18T00:00:02Z", ContentType: "text", ExtractionVersion: CurrentExtractionVersion, + }}), + }) + plan := recoveryDestinationPlanFromDBs(t, ctx, activePath, strandedPath) + activeSnapshot := snapshotRecoveryDB(t, activePath) + strandedSnapshot := snapshotRecoveryDB(t, strandedPath) + if got, want := len(plan.Records), 3; got != want { + t.Fatalf("planned records = %d, want %d", got, want) + } + if got, want := plan.ExactDuplicates, 1; got != want { + t.Fatalf("planned exact duplicates = %d, want %d", got, want) + } + + destPath, err := CreateRecoveryDestination(ctx, dir, plan) + if err != nil { + t.Fatalf("CreateRecoveryDestination: %v", err) + } + if destPath == activePath || destPath == strandedPath { + t.Fatalf("destination path %s reused an input path", destPath) + } + + assertRecoveryDestinationRecords(t, destPath, plan) + assertRecoveryDestinationFTS(t, destPath, "sentinelalpha", 1, "strandedtoolomega", 1) + assertRecoveryDBSnapshot(t, activePath, activeSnapshot) + assertRecoveryDBSnapshot(t, strandedPath, strandedSnapshot) +} + +func TestRecoveryDestinationStartsFreshAtCurrentSchema(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + sourcePath := filepath.Join(dir, "legacy-source.db") + createRecoveryDestinationSourceDB(t, sourcePath, []IndexedFile{ + recoveryDestinationIndexedFile("/sessions/source.jsonl", "source-hash", []IndexedMessage{{ + Ordinal: 0, Role: "user", Text: "fresh current schema sentinel", UUID: "11111111-1111-4111-8111-111111111111", + Timestamp: "2026-08-18T00:00:00Z", ContentType: "text", ExtractionVersion: CurrentExtractionVersion, + }}), + }) + plan := recoveryDestinationPlanFromDBs(t, ctx, sourcePath) + + destPath, err := CreateRecoveryDestination(ctx, dir, plan) + if err != nil { + t.Fatalf("CreateRecoveryDestination: %v", err) + } + if filepath.Dir(destPath) != dir || !strings.HasPrefix(filepath.Base(destPath), ".backscroll-recover-") { + t.Fatalf("destination path %s is not a sibling recovery temp in %s", destPath, dir) + } + info, err := os.Stat(destPath) + if err != nil { + t.Fatalf("stat destination: %v", err) + } + if got := info.Mode().Perm(); got != 0o600 { + t.Fatalf("destination permissions = %v, want 0600", got) + } + + db, err := OpenReadOnly(destPath) + if err != nil { + t.Fatalf("open destination readonly: %v", err) + } + defer func() { _ = db.Close() }() + assertCurrentShape(t, db.DB()) + if recoveryDestinationColumnExists(t, db, "search_items", "source_metadata") { + t.Fatal("destination has legacy source_metadata column; want fresh current schema") + } + var indexedFiles int + if err := db.DB().QueryRow(`SELECT COUNT(*) FROM indexed_files`).Scan(&indexedFiles); err != nil { + t.Fatalf("count indexed_files: %v", err) + } + if indexedFiles != 0 { + t.Fatalf("indexed_files rows = %d, want 0 in fresh recovery destination", indexedFiles) + } +} + +func TestRecoveryDestinationIndependentVerificationRejectsTamper(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + sourcePath := filepath.Join(dir, "source.db") + createRecoveryDestinationSourceDB(t, sourcePath, []IndexedFile{ + recoveryDestinationIndexedFile("/sessions/source.jsonl", "source-hash", []IndexedMessage{{ + Ordinal: 0, Role: "assistant", Text: "tamper detector sentinel", UUID: "11111111-1111-4111-8111-111111111111", + Timestamp: "2026-08-18T00:00:00Z", ContentType: "text", ExtractionVersion: CurrentExtractionVersion, + }}), + }) + plan := recoveryDestinationPlanFromDBs(t, ctx, sourcePath) + destPath, err := CreateRecoveryDestination(ctx, dir, plan) + if err != nil { + t.Fatalf("CreateRecoveryDestination: %v", err) + } + + mutateRecoveryDatabase(t, destPath, ` + INSERT INTO indexed_files(path, hash) + VALUES ('/tampered/invented-source.jsonl', 'invented-hash'); + `) + if err := VerifyRecoveryDestination(ctx, destPath, plan); err == nil { + t.Fatal("VerifyRecoveryDestination accepted a tampered destination with invented source accounting") + } +} + +func TestRecoveryDestinationVerificationRejectsOrphanedDerivedRows(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + sourcePath := filepath.Join(dir, "source.db") + createRecoveryDestinationSourceDB(t, sourcePath, []IndexedFile{ + recoveryDestinationIndexedFile("/sessions/source.jsonl", "source-hash", []IndexedMessage{{ + Ordinal: 0, Role: "user", Text: "foreign key tamper sentinel", UUID: "11111111-1111-4111-8111-111111111111", + Timestamp: "2026-08-18T00:00:00Z", ContentType: "text", ExtractionVersion: CurrentExtractionVersion, + }}), + }) + plan := recoveryDestinationPlanFromDBs(t, ctx, sourcePath) + destPath, err := CreateRecoveryDestination(ctx, dir, plan) + if err != nil { + t.Fatalf("CreateRecoveryDestination: %v", err) + } + + mutateRecoveryDatabase(t, destPath, ` + INSERT INTO template_matches(template_id, source_path, ordinal) + VALUES (999, '/sessions/source.jsonl', 0); + `) + if err := VerifyRecoveryDestination(ctx, destPath, plan); err == nil || !strings.Contains(err.Error(), "foreign_key_check") { + t.Fatalf("VerifyRecoveryDestination orphaned derived row error = %v, want foreign key diagnostics", err) + } +} + +func TestRecoveryDestinationVerificationRejectsDeletedRows(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + sourcePath := filepath.Join(dir, "source.db") + createRecoveryDestinationSourceDB(t, sourcePath, []IndexedFile{ + recoveryDestinationIndexedFile("/sessions/source.jsonl", "source-hash", []IndexedMessage{{ + Ordinal: 0, Role: "user", Text: "row deletion tamper sentinel", UUID: "11111111-1111-4111-8111-111111111111", + Timestamp: "2026-08-18T00:00:00Z", ContentType: "text", ExtractionVersion: CurrentExtractionVersion, + }}), + }) + plan := recoveryDestinationPlanFromDBs(t, ctx, sourcePath) + destPath, err := CreateRecoveryDestination(ctx, dir, plan) + if err != nil { + t.Fatalf("CreateRecoveryDestination: %v", err) + } + + mutateRecoveryDatabase(t, destPath, `DELETE FROM search_items;`) + if err := VerifyRecoveryDestination(ctx, destPath, plan); err == nil { + t.Fatal("VerifyRecoveryDestination accepted a destination with deleted recovery rows") + } +} + +func TestRecoveryDestinationVerificationRejectsEqualCountWrongIdentityAndPayload(t *testing.T) { + cases := []struct { + name string + mutate string + }{ + { + name: "identity", + mutate: `UPDATE search_items + SET uuid = '99999999-9999-4999-8999-999999999999' + WHERE source_path = '/sessions/source.jsonl' AND ordinal = 0;`, + }, + { + name: "payload", + mutate: `UPDATE search_items + SET role = 'assistant' + WHERE source_path = '/sessions/source.jsonl' AND ordinal = 0;`, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + sourcePath := filepath.Join(dir, "source.db") + createRecoveryDestinationSourceDB(t, sourcePath, []IndexedFile{ + recoveryDestinationIndexedFile("/sessions/source.jsonl", "source-hash", []IndexedMessage{{ + Ordinal: 0, Role: "user", Text: "equal count tamper sentinel", UUID: "11111111-1111-4111-8111-111111111111", + Timestamp: "2026-08-18T00:00:00Z", ContentType: "text", ExtractionVersion: CurrentExtractionVersion, + }}), + }) + plan := recoveryDestinationPlanFromDBs(t, ctx, sourcePath) + destPath, err := CreateRecoveryDestination(ctx, dir, plan) + if err != nil { + t.Fatalf("CreateRecoveryDestination: %v", err) + } + + mutateRecoveryDatabase(t, destPath, tc.mutate) + if err := VerifyRecoveryDestination(ctx, destPath, plan); err == nil { + t.Fatalf("VerifyRecoveryDestination accepted equal-count %s tampering", tc.name) + } + }) + } +} + +func TestRecoveryDestinationVerificationRejectsEqualCountWrongFTSSurface(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + sourcePath := filepath.Join(dir, "source.db") + createRecoveryDestinationSourceDB(t, sourcePath, []IndexedFile{ + recoveryDestinationIndexedFile("/sessions/prose.jsonl", "prose-hash", []IndexedMessage{ + {Ordinal: 0, Role: "user", Text: "first message keeps representativealpha", UUID: "11111111-1111-4111-8111-111111111111", ContentType: "text", ExtractionVersion: CurrentExtractionVersion}, + {Ordinal: 1, Role: "assistant", Text: "second message must stay messagesurfacebeta", UUID: "22222222-2222-4222-8222-222222222222", ContentType: "text", ExtractionVersion: CurrentExtractionVersion}, + }), + recoveryDestinationIndexedFile("/sessions/tool.jsonl", "tool-hash", []IndexedMessage{ + {Ordinal: 0, Role: "assistant", Text: "Bash command=toolonlysurfacegamma", UUID: "33333333-3333-4333-8333-333333333333", ContentType: "tool", ToolName: "Bash", CommandHead: "toolonlysurfacegamma", ExtractionVersion: CurrentExtractionVersion}, + }), + }) + plan := recoveryDestinationPlanFromDBs(t, ctx, sourcePath) + destPath, err := CreateRecoveryDestination(ctx, dir, plan) + if err != nil { + t.Fatalf("CreateRecoveryDestination: %v", err) + } + + secondMessageID := recoveryDestinationRowID(t, destPath, "/sessions/prose.jsonl", 1) + toolID := recoveryDestinationRowID(t, destPath, "/sessions/tool.jsonl", 0) + mutateRecoveryDatabase(t, destPath, ` + INSERT INTO messages_fts(messages_fts, rowid, text) + SELECT 'delete', id, text FROM search_items WHERE id = `+strconv.FormatInt(secondMessageID, 10)+`; + INSERT INTO messages_fts(rowid, text) + SELECT id, text FROM search_items WHERE id = `+strconv.FormatInt(toolID, 10)+`; + `) + if err := VerifyRecoveryDestination(ctx, destPath, plan); err == nil { + t.Fatal("VerifyRecoveryDestination accepted equal-count tampering with a tool row on messages_fts") + } +} + +func TestRecoveryDestinationAcceptsPathOrdinalIdentityWithNilAndEmptyUUID(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + emptyUUID := "" + plan := compat.RecoveryPlan{Records: []compat.CanonicalRecord{ + {Record: models.IndexedRecord{Source: "session", SourcePath: "/sessions/nil-uuid.jsonl", Ordinal: 0, Role: "user", Text: "nil uuid ordinal zero messagealpha", UUID: nil, ContentType: "text"}}, + {Record: models.IndexedRecord{Source: "session", SourcePath: "/sessions/empty-uuid-a.jsonl", Ordinal: 0, Role: "assistant", Text: "empty uuid ordinal zero toolbeta", UUID: &emptyUUID, ContentType: "tool"}}, + {Record: models.IndexedRecord{Source: "session", SourcePath: "/sessions/empty-uuid-b.jsonl", Ordinal: 1, Role: "assistant", Text: "second empty uuid distinct path ordinal gammabeta", UUID: &emptyUUID, ContentType: "text"}}, + }} + + destPath, err := CreateRecoveryDestination(ctx, dir, plan) + if err != nil { + t.Fatalf("CreateRecoveryDestination with path/ordinal identities: %v", err) + } + if err := VerifyRecoveryDestination(ctx, destPath, plan); err != nil { + t.Fatalf("VerifyRecoveryDestination with path/ordinal identities: %v", err) + } +} + +func TestRecoveryDestinationAcceptsUntokenizedRecordsPersistently(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + plan := compat.RecoveryPlan{Records: []compat.CanonicalRecord{ + {Record: models.IndexedRecord{Source: "session", SourcePath: "/sessions/punctuation.jsonl", Ordinal: 0, Role: "user", Text: "!!", ContentType: "text"}}, + {Record: models.IndexedRecord{Source: "session", SourcePath: "/sessions/tool-punctuation.jsonl", Ordinal: 1, Role: "assistant", Text: "??", ContentType: "tool"}}, + }} + + destPath, err := CreateRecoveryDestination(ctx, dir, plan) + if err != nil { + t.Fatalf("CreateRecoveryDestination with untokenized records: %v", err) + } + if err := VerifyRecoveryDestination(ctx, destPath, plan); err != nil { + t.Fatalf("VerifyRecoveryDestination with untokenized records: %v", err) + } + assertRecoveryDestinationRecords(t, destPath, plan) + assertRecoveryDestinationFTS(t, destPath, "punctuationabsent", 0, "toolabsent", 0) +} + +func TestRecoveryDestinationRejectsUnsafePlanIdentitiesAndCleansUp(t *testing.T) { + ctx := context.Background() + cases := []struct { + name string + record models.IndexedRecord + }{ + { + name: "invalid_uuid", + record: models.IndexedRecord{Source: "session", SourcePath: "/sessions/invalid-uuid.jsonl", Ordinal: 0, Role: "user", Text: "invalid uuid record", UUID: stringPtr("not-a-uuid"), ContentType: "text"}, + }, + { + name: "missing_fallback_identity", + record: models.IndexedRecord{Source: "session", SourcePath: "", Ordinal: -1, Role: "assistant", Text: "missing fallback identity", ContentType: "text"}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + plan := compat.RecoveryPlan{Records: []compat.CanonicalRecord{{Record: tc.record}}} + if _, err := CreateRecoveryDestination(ctx, dir, plan); err == nil { + t.Fatalf("CreateRecoveryDestination accepted unsafe plan identity %+v", tc.record) + } + assertNoRecoveryDestinationTemps(t, dir) + }) + } +} + +func TestRecoveryDestinationRejectsDuplicatePathOrdinalIdentityAndCleansUp(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + emptyUUID := "" + plan := compat.RecoveryPlan{Records: []compat.CanonicalRecord{ + {Record: models.IndexedRecord{Source: "session", SourcePath: "/sessions/duplicate-path.jsonl", Ordinal: 0, Role: "user", Text: "first duplicate path ordinal", UUID: nil, ContentType: "text"}}, + {Record: models.IndexedRecord{Source: "session", SourcePath: "/sessions/duplicate-path.jsonl", Ordinal: 0, Role: "assistant", Text: "second duplicate path ordinal", UUID: &emptyUUID, ContentType: "text"}}, + }} + + if _, err := CreateRecoveryDestination(ctx, dir, plan); err == nil { + t.Fatal("CreateRecoveryDestination accepted duplicate path/ordinal identities") + } + assertNoRecoveryDestinationTemps(t, dir) +} + +func TestRecoverConflictOrUninterpretableRollsBackEverything(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + sourcePath := filepath.Join(dir, "active.db") + createRecoveryDestinationSourceDB(t, sourcePath, []IndexedFile{ + recoveryDestinationIndexedFile("/sessions/active.jsonl", "active-hash", []IndexedMessage{{ + Ordinal: 0, Role: "user", Text: "safe source remains untouched", UUID: "11111111-1111-4111-8111-111111111111", + Timestamp: "2026-08-18T00:00:00Z", ContentType: "text", ExtractionVersion: CurrentExtractionVersion, + }}), + }) + plan := recoveryDestinationPlanFromDBs(t, ctx, sourcePath) + sourceSnapshot := snapshotRecoveryDB(t, sourcePath) + conflictUUID := "22222222-2222-4222-8222-222222222222" + plan.Records = append(plan.Records, + compat.CanonicalRecord{Record: models.IndexedRecord{Source: "session", SourcePath: "/sessions/conflict-a.jsonl", Ordinal: 0, Role: "user", Text: "first conflicting payload", UUID: &conflictUUID, ContentType: "text"}}, + compat.CanonicalRecord{Record: models.IndexedRecord{Source: "session", SourcePath: "/sessions/conflict-b.jsonl", Ordinal: 1, Role: "assistant", Text: "second conflicting payload", UUID: &conflictUUID, ContentType: "text"}}, + ) + + _, err := CreateRecoveryDestination(ctx, dir, plan) + if err == nil { + t.Fatal("CreateRecoveryDestination succeeded for a plan with conflicting import identities") + } + assertNoRecoveryDestinationTemps(t, dir) + assertRecoveryDBSnapshot(t, sourcePath, sourceSnapshot) + + diagnosticBearingPlan := plan + diagnosticBearingPlan.Records = nil + _, err = CreateRecoveryDestination(ctx, dir, diagnosticBearingPlan) + if err == nil { + t.Fatal("CreateRecoveryDestination succeeded for an uninterpretable/diagnostic-bearing plan") + } + assertNoRecoveryDestinationTemps(t, dir) + assertRecoveryDBSnapshot(t, sourcePath, sourceSnapshot) +} + +func createRecoveryDestinationSourceDB(t *testing.T, path string, files []IndexedFile) { + t.Helper() + db, err := Open(path) + if err != nil { + t.Fatalf("open source database %s: %v", path, err) + } + if err := db.SyncFiles(files); err != nil { + _ = db.Close() + t.Fatalf("sync source database %s: %v", path, err) + } + if err := db.Close(); err != nil { + t.Fatalf("close source database %s: %v", path, err) + } +} + +func recoveryDestinationIndexedFile(path, hash string, messages []IndexedMessage) IndexedFile { + return IndexedFile{SourcePath: path, Source: "session", Hash: hash, Project: "project", Messages: messages} +} + +func recoveryDestinationPlanFromDBs(t *testing.T, ctx context.Context, paths ...string) compat.RecoveryPlan { + t.Helper() + inputs := make([]compat.RecoveryInput, 0, len(paths)) + for _, path := range paths { + db, err := OpenReadOnly(path) + if err != nil { + t.Fatalf("open readonly %s: %v", path, err) + } + input, diag, err := ReadRecoveryInput(ctx, db) + closeErr := db.Close() + if err != nil || diag != nil { + t.Fatalf("ReadRecoveryInput %s err=%v diag=%+v", path, err, diag) + } + if closeErr != nil { + t.Fatalf("close readonly %s: %v", path, closeErr) + } + inputs = append(inputs, input) + } + plan, diagnostics, err := compat.PlanRecovery(inputs) + if err != nil { + t.Fatalf("PlanRecovery: %v", err) + } + if len(diagnostics) != 0 { + t.Fatalf("PlanRecovery diagnostics: %+v", diagnostics) + } + return plan +} + +func assertRecoveryDestinationRecords(t *testing.T, path string, plan compat.RecoveryPlan) { + t.Helper() + db, err := OpenReadOnly(path) + if err != nil { + t.Fatalf("open destination readonly: %v", err) + } + defer func() { _ = db.Close() }() + input, diag, err := ReadRecoveryInput(context.Background(), db) + if err != nil || diag != nil { + t.Fatalf("ReadRecoveryInput destination err=%v diag=%+v", err, diag) + } + got := append([]models.IndexedRecord(nil), input.Records...) + want := make([]models.IndexedRecord, 0, len(plan.Records)) + for _, planned := range plan.Records { + want = append(want, planned.Record) + } + sortRecoveryDestinationRecords(got) + sortRecoveryDestinationRecords(want) + if !reflect.DeepEqual(got, want) { + t.Fatalf("destination records mismatch\ngot: %#v\nwant: %#v", got, want) + } + var indexedFiles int + if err := db.DB().QueryRow(`SELECT COUNT(*) FROM indexed_files`).Scan(&indexedFiles); err != nil { + t.Fatalf("count indexed_files: %v", err) + } + if indexedFiles != 0 { + t.Fatalf("indexed_files rows = %d, want 0; recovery must not invent source hashes", indexedFiles) + } +} + +func assertRecoveryDestinationFTS(t *testing.T, path, msgTerm string, wantMsg int, toolTerm string, wantTool int) { + t.Helper() + db, err := OpenReadOnly(path) + if err != nil { + t.Fatalf("open destination readonly for FTS: %v", err) + } + defer func() { _ = db.Close() }() + var msgHits, toolHits int + if err := db.DB().QueryRow(`SELECT COUNT(*) FROM messages_fts WHERE messages_fts MATCH ?`, msgTerm).Scan(&msgHits); err != nil { + t.Fatalf("query messages_fts: %v", err) + } + if err := db.DB().QueryRow(`SELECT COUNT(*) FROM tool_fts WHERE tool_fts MATCH ?`, toolTerm).Scan(&toolHits); err != nil { + t.Fatalf("query tool_fts: %v", err) + } + if msgHits != wantMsg || toolHits != wantTool { + t.Fatalf("FTS hits messages=%d tool=%d, want messages=%d tool=%d", msgHits, toolHits, wantMsg, wantTool) + } +} + +func recoveryDestinationColumnExists(t *testing.T, db *Database, table, column string) bool { + t.Helper() + rows, err := db.DB().Query(`PRAGMA table_info(` + table + `)`) + if err != nil { + t.Fatalf("table_info %s: %v", table, err) + } + defer rows.Close() + for rows.Next() { + var cid int + var name, typ string + var notNull, pk int + var defaultValue any + if err := rows.Scan(&cid, &name, &typ, ¬Null, &defaultValue, &pk); err != nil { + t.Fatalf("scan table_info %s: %v", table, err) + } + if name == column { + return true + } + } + if err := rows.Err(); err != nil { + t.Fatalf("read table_info %s: %v", table, err) + } + return false +} + +func assertNoRecoveryDestinationTemps(t *testing.T, dir string) { + t.Helper() + matches, err := filepath.Glob(filepath.Join(dir, ".backscroll-recover-*.db*")) + if err != nil { + t.Fatal(err) + } + kept := matches[:0] + for _, match := range matches { + base := filepath.Base(match) + if strings.HasPrefix(base, ".backscroll-recover-") { + kept = append(kept, match) + } + } + if len(kept) != 0 { + t.Fatalf("recovery destination temp files remain: %v", kept) + } +} + +func recoveryDestinationRowID(t *testing.T, path, sourcePath string, ordinal int64) int64 { + t.Helper() + db, err := OpenReadOnly(path) + if err != nil { + t.Fatalf("open destination readonly for row id: %v", err) + } + defer func() { _ = db.Close() }() + var id int64 + if err := db.DB().QueryRow(`SELECT id FROM search_items WHERE source_path = ? AND ordinal = ?`, sourcePath, ordinal).Scan(&id); err != nil { + t.Fatalf("lookup row id for %s ordinal %d: %v", sourcePath, ordinal, err) + } + return id +} + +func sortRecoveryDestinationRecords(records []models.IndexedRecord) { + sort.Slice(records, func(i, j int) bool { + if records[i].SourcePath != records[j].SourcePath { + return records[i].SourcePath < records[j].SourcePath + } + if records[i].Ordinal != records[j].Ordinal { + return records[i].Ordinal < records[j].Ordinal + } + if pointerTestString(records[i].UUID) != pointerTestString(records[j].UUID) { + return pointerTestString(records[i].UUID) < pointerTestString(records[j].UUID) + } + return records[i].Text < records[j].Text + }) +} + +func pointerTestString(value *string) string { + if value == nil { + return "" + } + return *value +} diff --git a/internal/storage/recovery_records.go b/internal/storage/recovery_records.go new file mode 100644 index 0000000..086f51e --- /dev/null +++ b/internal/storage/recovery_records.go @@ -0,0 +1,143 @@ +package storage + +import ( + "context" + "database/sql" + "errors" + "fmt" + + "github.com/pablontiv/backscroll/internal/compat" + "github.com/pablontiv/backscroll/internal/models" +) + +// ReadRecoveryInput reads every recoverable search_items row from a supported +// historical index lineage without applying migrations or mutating the database. +func ReadRecoveryInput(ctx context.Context, db *Database) (compat.RecoveryInput, *compat.Diagnostic, error) { + return ReadRecoveryInputFromQueryer(ctx, db.DB()) +} + +// ReadRecoveryInputFromQueryer reads recoverable rows through q without opening +// another connection. Recovery apply uses this while holding a SQLite write +// reservation, so the final active input and replacement plan are derived from +// the same reserved snapshot. +func ReadRecoveryInputFromQueryer(ctx context.Context, q compat.Queryer) (compat.RecoveryInput, *compat.Diagnostic, error) { + plan, diag, err := compat.InspectIndex(ctx, q) + if err != nil || diag != nil { + return compat.RecoveryInput{}, diag, err + } + + records, diag, err := readRecordsForSignature(ctx, q, plan.From.Signature) + if err != nil || diag != nil { + return compat.RecoveryInput{}, diag, err + } + return compat.RecoveryInput{Shape: plan.From, Records: records, RowCount: len(records)}, nil, nil +} + +func readRecordsForSignature(ctx context.Context, q compat.Queryer, signature string) ([]models.IndexedRecord, *compat.Diagnostic, error) { + switch signature { + case + "sha256:e2f7cd4bd71c964717c00fd67c2b3f396306307f578382c4a7956f83f8555a57", // v1 + "sha256:a4256a6029bbc953df37a12979fc81b1879ef9f8ba28ff5cfe6b78472dee804b", // v2 + "sha256:ed864d83d4c873bd34e0e3708f32a5bf47d65cac8ad55ff04305a81d91e9e22e", // v3 + "sha256:a7b05ddcb786f8d633fd2f27b19e0274fdf78dbd5c0e5ef420cecc831bcc3a8f", // v3 without source_metadata + "sha256:b2bba3cec75bb3f6c697e78882e3924550832e3b5706d013f3f54d32a1c25acd", // v4 + "sha256:ce23ee41f1af6007bd0bde7b020f2cac4c215d23bae456bdfce25b90313c709e", // v5 with source_metadata + "sha256:95c1c0aa96f1093511dd4e159ec3b574aacdfc67136531b2fb9c3cd01e90aad0", // v5 without source_metadata + "sha256:68f237fe4a85c97df36522ebdd54703afb97ea84c3f0b0cd45e6400c5e95e220", // v6 + "sha256:dcaa1205df039fa545aa7ebe672d391acfcbf651869c2603ceef12dab9de00d2", // v7 + "sha256:6d503881ebac89e009644df5bf24cf7c4b1afed334afb37e6ee87d2ca6edae87", // v8 + "sha256:e41ae857c862ffa10516681b65bcd8c755484b338a6847dad5c0d770a202670f", // v9 + "sha256:8e28ce0fe1c5f3cd36f2d64ac7a7996f032e66c0c32468a0a206eb983491388c", // v10 + "sha256:975656bb5e894e12bd30aa65bb3366ca2bdeee23f59aca8e133b18a62e7ffad5", // v11 + "sha256:ff136d58048d69f02be857f632750ef3e2daa35fd6ba637f7645986950f83d31", // v12 + "sha256:ef52be2fb56acd0e51b38506746fa3594ed1de76bb0b5dd180767d3c903fb46d", // v13 canonical + "sha256:19d65765b8b0d0bb7d1c41692712c395627e005b7aeccca5f887c75be781e314", // v13 legacy ALTER-built alias + "sha256:f52b4132322f3addb0fc01304f92ca6a2c2f4706e5ac010c59a5ea594cbd25b2": // v13 legacy schema_migrations alias + return readCanonicalSearchItems(ctx, q) + default: + return nil, &compat.Diagnostic{ + Code: compat.CodeUnsupportedLineage, + Summary: fmt.Sprintf("unsupported index schema %s", signature), + }, nil + } +} + +func readCanonicalSearchItems(ctx context.Context, q compat.Queryer) ([]models.IndexedRecord, *compat.Diagnostic, error) { + rows, err := q.QueryContext(ctx, ` + SELECT source, source_path, ordinal, role, text, project, uuid, timestamp, content_type + FROM search_items + ORDER BY source_path, ordinal, id + `) + if err != nil { + return nil, nil, fmt.Errorf("query recovery records: %w", err) + } + return readCanonicalSearchItemsFromRows(rows) +} + +type recoveryRows interface { + Next() bool + Scan(dest ...any) error + Err() error + Close() error +} + +func readCanonicalSearchItemsFromRows(rows recoveryRows) (records []models.IndexedRecord, diag *compat.Diagnostic, err error) { + defer func() { + if closeErr := rows.Close(); closeErr != nil { + err = errors.Join(err, fmt.Errorf("close recovery records: %w", closeErr)) + } + }() + + for rows.Next() { + record, rowDiag := scanRecoveryRecord(rows) + if rowDiag != nil { + return nil, rowDiag, nil + } + records = append(records, record) + } + if err := rows.Err(); err != nil { + return nil, nil, fmt.Errorf("read recovery records: %w", err) + } + return records, nil, nil +} + +type recoveryScanner interface { + Scan(dest ...any) error +} + +func scanRecoveryRecord(scanner recoveryScanner) (models.IndexedRecord, *compat.Diagnostic) { + var source, sourcePath, role, text, contentType sql.NullString + var project, uuid, timestamp sql.NullString + var ordinal sql.NullInt64 + if err := scanner.Scan(&source, &sourcePath, &ordinal, &role, &text, &project, &uuid, ×tamp, &contentType); err != nil { + return models.IndexedRecord{}, uninterpretableRecoveryRowDiagnostic() + } + if !source.Valid || !sourcePath.Valid || !ordinal.Valid || !role.Valid || !text.Valid || !contentType.Valid { + return models.IndexedRecord{}, uninterpretableRecoveryRowDiagnostic() + } + return models.IndexedRecord{ + Source: source.String, + SourcePath: sourcePath.String, + Ordinal: ordinal.Int64, + Role: role.String, + Text: text.String, + Project: recoveryStringPtr(project), + UUID: recoveryStringPtr(uuid), + Timestamp: recoveryStringPtr(timestamp), + ContentType: contentType.String, + }, nil +} + +func recoveryStringPtr(value sql.NullString) *string { + if !value.Valid { + return nil + } + return &value.String +} + +func uninterpretableRecoveryRowDiagnostic() *compat.Diagnostic { + return &compat.Diagnostic{ + Code: compat.CodeUninterpretableRow, + Summary: "index contains a row that cannot be interpreted as a canonical recovery record", + } +} diff --git a/internal/storage/recovery_records_test.go b/internal/storage/recovery_records_test.go new file mode 100644 index 0000000..c02e269 --- /dev/null +++ b/internal/storage/recovery_records_test.go @@ -0,0 +1,414 @@ +package storage + +import ( + "bytes" + "context" + "database/sql" + "errors" + "fmt" + "os" + "path/filepath" + "reflect" + "sort" + "strings" + "testing" + "time" + + "github.com/pablontiv/backscroll/internal/compat" + "github.com/pablontiv/backscroll/internal/models" +) + +type recoveryFixtureHandle struct { + db *Database + dbPath string + dbSnapshot recoveryDBSnapshot + sourcePath string + sourceSnapshot recoveryFileSnapshot +} + +type recoveryFileSnapshot struct { + data []byte + modTime time.Time +} + +type recoveryDBSnapshot struct { + main recoveryFileSnapshot + sidecars map[string]recoveryFileSnapshot +} + +func TestReadRecoveryInputAdaptsSupportedLineages(t *testing.T) { + for _, fixture := range []string{"active-v13.sql", "stranded-v3-no-source-metadata.sql", "stranded-v7.sql"} { + t.Run(fixture, func(t *testing.T) { + handle := openRecoveryFixtureReadOnly(t, fixture) + defer func() { _ = handle.db.Close() }() + + got, diag, err := ReadRecoveryInput(context.Background(), handle.db) + if err != nil || diag != nil { + t.Fatalf("ReadRecoveryInput err=%v diagnostic=%+v", err, diag) + } + if got.RowCount != 2 || got.RowCount != len(got.Records) { + t.Fatalf("input row count = %d records = %d, want 2", got.RowCount, len(got.Records)) + } + if got.Shape.Signature == "" || got.Shape.AppliedVersion == 0 { + t.Fatalf("shape was not populated from inspection: %+v", got.Shape) + } + assertRecoveryRecordsEqual(t, got.Records, expectedRecoveryRecords(fixture)) + assertQueryOnly(t, handle.db, true) + assertRecoveryFixtureUnchanged(t, handle) + }) + } +} + +func TestReadRecoveryInputSupportsCatalogReadableSignatures(t *testing.T) { + fixtures, err := filepath.Glob(filepath.Join("..", "compat", "testdata", "release-schemas", "v*.sql")) + if err != nil { + t.Fatal(err) + } + if len(fixtures) == 0 { + t.Fatal("no catalog schema fixtures found") + } + sort.Strings(fixtures) + + for _, fixturePath := range fixtures { + fixture := filepath.Base(fixturePath) + t.Run(fixture, func(t *testing.T) { + dbPath := buildRecoveryDatabaseFromSQLFile(t, fixturePath) + slug := strings.TrimSuffix(fixture, ".sql") + insertRecoverySentinel(t, dbPath, slug) + handle := openRecoveryDBReadOnly(t, dbPath, fixturePath) + defer func() { _ = handle.db.Close() }() + + got, diag, err := ReadRecoveryInput(context.Background(), handle.db) + if err != nil || diag != nil { + t.Fatalf("ReadRecoveryInput err=%v diagnostic=%+v", err, diag) + } + if got.RowCount != 1 || len(got.Records) != 1 { + t.Fatalf("row count = %d records = %d, want 1", got.RowCount, len(got.Records)) + } + assertRecoveryRecordsEqual(t, got.Records, []models.IndexedRecord{expectedCatalogRecoveryRecord(slug)}) + assertQueryOnly(t, handle.db, true) + assertRecoveryFixtureUnchanged(t, handle) + }) + } +} + +func TestReadRecoveryInputRejectsUnknownShape(t *testing.T) { + dbPath := buildRecoveryFixtureDatabase(t, "active-v13.sql") + mutateRecoveryDatabase(t, dbPath, `CREATE TABLE recovery_unknown_shape (id INTEGER PRIMARY KEY);`) + handle := openRecoveryDBReadOnly(t, dbPath, recoveryFixturePath("active-v13.sql")) + defer func() { _ = handle.db.Close() }() + + got, diag, err := ReadRecoveryInput(context.Background(), handle.db) + if err != nil { + t.Fatalf("ReadRecoveryInput error = %v", err) + } + if diag == nil || diag.Code != compat.CodeUnsupportedLineage { + t.Fatalf("diagnostic = %+v, want CodeUnsupportedLineage", diag) + } + if len(diag.Continuation) != 0 { + t.Fatalf("diagnostic continuation = %v, want empty", diag.Continuation) + } + if got.RowCount != 0 || len(got.Records) != 0 { + t.Fatalf("input = %+v, want empty on unsupported lineage", got) + } + assertQueryOnly(t, handle.db, true) + assertRecoveryFixtureUnchanged(t, handle) +} + +func TestReadRecoveryInputRejectsMissingCanonicalPayload(t *testing.T) { + dbPath := buildRecoveryFixtureDatabase(t, "active-v13.sql") + mutateRecoveryDatabase(t, dbPath, ` + INSERT INTO search_items (source, source_path, ordinal, role, text, content_type) + VALUES ('session', '/fixtures/recovery/corrupt.jsonl', 'not-an-integer', 'user', 'corrupt ordinal sentinel', 'text'); + `) + handle := openRecoveryDBReadOnly(t, dbPath, recoveryFixturePath("active-v13.sql")) + defer func() { _ = handle.db.Close() }() + + got, diag, err := ReadRecoveryInput(context.Background(), handle.db) + if err != nil { + t.Fatalf("ReadRecoveryInput error = %v", err) + } + if diag == nil || diag.Code != compat.CodeUninterpretableRow { + t.Fatalf("diagnostic = %+v, want CodeUninterpretableRow", diag) + } + if len(diag.Continuation) != 0 { + t.Fatalf("diagnostic continuation = %v, want empty", diag.Continuation) + } + if got.RowCount != 0 || len(got.Records) != 0 { + t.Fatalf("input = %+v, want empty on corrupt row", got) + } + assertQueryOnly(t, handle.db, true) + assertRecoveryFixtureUnchanged(t, handle) +} + +func TestReadCanonicalSearchItemsPropagatesRowsCloseError(t *testing.T) { + closeErr := errors.New("injected recovery rows close failure") + _, diag, err := readCanonicalSearchItemsFromRows(&fakeRecoveryRows{closeErr: closeErr}) + if diag != nil { + t.Fatalf("diagnostic = %+v, want nil", diag) + } + if err == nil || !errors.Is(err, closeErr) { + t.Fatalf("readCanonicalSearchItemsFromRows error = %v, want close error", err) + } +} + +func TestReadCanonicalSearchItemsJoinsRowsReadAndCloseErrors(t *testing.T) { + readErr := errors.New("injected recovery rows read failure") + closeErr := errors.New("injected recovery rows close failure") + _, diag, err := readCanonicalSearchItemsFromRows(&fakeRecoveryRows{err: readErr, closeErr: closeErr}) + if diag != nil { + t.Fatalf("diagnostic = %+v, want nil", diag) + } + if err == nil || !errors.Is(err, readErr) || !errors.Is(err, closeErr) { + t.Fatalf("readCanonicalSearchItemsFromRows error = %v, want read and close errors", err) + } +} + +type fakeRecoveryRows struct { + err error + closeErr error +} + +func (r *fakeRecoveryRows) Next() bool { return false } +func (r *fakeRecoveryRows) Scan(dest ...any) error { return nil } +func (r *fakeRecoveryRows) Err() error { return r.err } +func (r *fakeRecoveryRows) Close() error { return r.closeErr } + +func TestReadRecoveryInputPerformsNoWrites(t *testing.T) { + for _, fixture := range []string{"active-v13.sql", "stranded-v3-no-source-metadata.sql", "stranded-v7.sql"} { + t.Run(fixture, func(t *testing.T) { + handle := openRecoveryFixtureReadOnly(t, fixture) + defer func() { _ = handle.db.Close() }() + + if _, diag, err := ReadRecoveryInput(context.Background(), handle.db); err != nil || diag != nil { + t.Fatalf("ReadRecoveryInput err=%v diagnostic=%+v", err, diag) + } + assertQueryOnly(t, handle.db, true) + assertRecoveryFixtureUnchanged(t, handle) + }) + } +} + +func expectedRecoveryRecords(fixture string) []models.IndexedRecord { + slug := strings.TrimSuffix(fixture, ".sql") + project := "project-" + slug + uuid := "uuid-" + slug + timestamp := "2026-08-18T12:34:56Z" + return []models.IndexedRecord{ + { + Source: "session", + SourcePath: "/fixtures/recovery/" + slug + "-defaults.jsonl", + Ordinal: 42, + Role: "user", + Text: "default sentinel for " + slug, + ContentType: "text", + }, + { + Source: "source-" + slug, + SourcePath: "/fixtures/recovery/" + slug + ".jsonl", + Ordinal: 41, + Role: "assistant", + Text: "text sentinel for " + slug, + Project: &project, + UUID: &uuid, + Timestamp: ×tamp, + ContentType: "content/" + slug, + }, + } +} + +func expectedCatalogRecoveryRecord(slug string) models.IndexedRecord { + project := "project-" + slug + uuid := "uuid-" + slug + timestamp := "2026-08-18T12:34:56Z" + return models.IndexedRecord{ + Source: "source-" + slug, + SourcePath: "/fixtures/recovery/" + slug + ".jsonl", + Ordinal: 7, + Role: "assistant", + Text: "catalog sentinel for " + slug, + Project: &project, + UUID: &uuid, + Timestamp: ×tamp, + ContentType: "content/" + slug, + } +} + +func assertRecoveryRecordsEqual(t *testing.T, got, want []models.IndexedRecord) { + t.Helper() + if !reflect.DeepEqual(got, want) { + t.Fatalf("records mismatch\ngot: %#v\nwant: %#v", got, want) + } +} + +func openRecoveryFixtureReadOnly(t *testing.T, fixture string) recoveryFixtureHandle { + t.Helper() + sourcePath := recoveryFixturePath(fixture) + dbPath := buildRecoveryFixtureDatabase(t, fixture) + return openRecoveryDBReadOnly(t, dbPath, sourcePath) +} + +func recoveryFixturePath(fixture string) string { + return filepath.Join("..", "..", "tests", "fixtures", "recovery", fixture) +} + +func buildRecoveryFixtureDatabase(t *testing.T, fixture string) string { + t.Helper() + return buildRecoveryDatabaseFromSQLFile(t, recoveryFixturePath(fixture)) +} + +func buildRecoveryDatabaseFromSQLFile(t *testing.T, fixturePath string) string { + t.Helper() + data, err := os.ReadFile(fixturePath) + if err != nil { + t.Fatal(err) + } + + dbPath := filepath.Join(t.TempDir(), "backscroll.db") + db, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatal(err) + } + defer func() { _ = db.Close() }() + if _, err := db.Exec(string(data)); err != nil { + t.Fatalf("load fixture %s: %v", fixturePath, err) + } + return dbPath +} + +func insertRecoverySentinel(t *testing.T, dbPath, slug string) { + t.Helper() + mutateRecoveryDatabase(t, dbPath, fmt.Sprintf(` + INSERT INTO indexed_files (path, hash, last_indexed) + VALUES ('/fixtures/recovery/%[1]s.jsonl', 'hash-%[1]s', '2026-08-18T00:00:00Z'); + INSERT INTO search_items (source, source_path, ordinal, role, text, timestamp, uuid, project, content_type) + VALUES ('source-%[1]s', '/fixtures/recovery/%[1]s.jsonl', 7, 'assistant', 'catalog sentinel for %[1]s', '2026-08-18T12:34:56Z', 'uuid-%[1]s', 'project-%[1]s', 'content/%[1]s'); + `, slug)) +} + +func mutateRecoveryDatabase(t *testing.T, dbPath string, stmt string) { + t.Helper() + db, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatal(err) + } + defer func() { _ = db.Close() }() + if _, err := db.Exec(stmt); err != nil { + t.Fatal(err) + } +} + +func openRecoveryDBReadOnly(t *testing.T, dbPath, sourcePath string) recoveryFixtureHandle { + t.Helper() + handle := recoveryFixtureHandle{ + dbPath: dbPath, + dbSnapshot: snapshotRecoveryDB(t, dbPath), + sourcePath: sourcePath, + sourceSnapshot: snapshotRecoveryFile(t, sourcePath), + } + db, err := OpenReadOnly(dbPath) + if err != nil { + t.Fatalf("OpenReadOnly: %v", err) + } + db.DB().SetMaxOpenConns(1) + if _, err := db.DB().Exec("PRAGMA query_only = ON"); err != nil { + _ = db.Close() + t.Fatalf("enable query_only: %v", err) + } + handle.db = db + assertQueryOnly(t, handle.db, true) + return handle +} + +func assertQueryOnly(t *testing.T, db *Database, want bool) { + t.Helper() + var gotInt int + if err := db.DB().QueryRow("PRAGMA query_only").Scan(&gotInt); err != nil { + t.Fatalf("query PRAGMA query_only: %v", err) + } + got := gotInt != 0 + if got != want { + t.Fatalf("PRAGMA query_only = %v, want %v", got, want) + } +} + +func assertRecoveryFixtureUnchanged(t *testing.T, handle recoveryFixtureHandle) { + t.Helper() + assertRecoveryFileSnapshot(t, "source SQL fixture", handle.sourcePath, handle.sourceSnapshot) + assertRecoveryDBSnapshot(t, handle.dbPath, handle.dbSnapshot) +} + +func assertRecoveryDBSnapshot(t *testing.T, dbPath string, want recoveryDBSnapshot) { + t.Helper() + assertRecoveryFileSnapshot(t, "database file", dbPath, want.main) + got := snapshotRecoverySidecars(t, dbPath) + if !reflect.DeepEqual(sortedMapKeys(got), sortedMapKeys(want.sidecars)) { + t.Fatalf("sidecar inventory = %v, want %v", sortedMapKeys(got), sortedMapKeys(want.sidecars)) + } + for name, wantSnapshot := range want.sidecars { + gotSnapshot := got[name] + if !bytes.Equal(gotSnapshot.data, wantSnapshot.data) || !gotSnapshot.modTime.Equal(wantSnapshot.modTime) { + t.Fatalf("sidecar %s changed", name) + } + } +} + +func snapshotRecoveryDB(t *testing.T, dbPath string) recoveryDBSnapshot { + t.Helper() + return recoveryDBSnapshot{ + main: snapshotRecoveryFile(t, dbPath), + sidecars: snapshotRecoverySidecars(t, dbPath), + } +} + +func snapshotRecoverySidecars(t *testing.T, dbPath string) map[string]recoveryFileSnapshot { + t.Helper() + dir := filepath.Dir(dbPath) + base := filepath.Base(dbPath) + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + snapshots := map[string]recoveryFileSnapshot{} + for _, entry := range entries { + name := entry.Name() + if !strings.HasPrefix(name, base+"-") { + continue + } + snapshots[name] = snapshotRecoveryFile(t, filepath.Join(dir, name)) + } + return snapshots +} + +func assertRecoveryFileSnapshot(t *testing.T, label, path string, want recoveryFileSnapshot) { + t.Helper() + got := snapshotRecoveryFile(t, path) + if !bytes.Equal(got.data, want.data) { + t.Fatalf("%s bytes changed for %s", label, path) + } + if !got.modTime.Equal(want.modTime) { + t.Fatalf("%s mtime changed for %s: got %s want %s", label, path, got.modTime, want.modTime) + } +} + +func snapshotRecoveryFile(t *testing.T, path string) recoveryFileSnapshot { + t.Helper() + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + return recoveryFileSnapshot{data: data, modTime: info.ModTime()} +} + +func sortedMapKeys[V any](m map[string]V) []string { + keys := make([]string, 0, len(m)) + for key := range m { + keys = append(keys, key) + } + sort.Strings(keys) + return keys +} diff --git a/internal/storage/storage.go b/internal/storage/storage.go index d2c615f..33d7c0e 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -1,34 +1,74 @@ package storage import ( + "context" "database/sql" + "errors" "fmt" + "io/fs" "os" + "path/filepath" _ "modernc.org/sqlite" + "github.com/pablontiv/backscroll/internal/compat" "github.com/pablontiv/backscroll/internal/embedding" ) // Database represents a SQLite database connection with FTS5 support. type Database struct { db *sql.DB + path string embeddingProvider embedding.EmbeddingProvider } +var openCompatibleApplyMigrationPlan = func(db *Database, ctx context.Context, plan compat.MigrationPlan) error { + return db.ApplyMigrationPlan(ctx, plan) +} + +var ErrImmutableReadOnlyWALUnsafe = errors.New("non-empty WAL makes immutable read-only content unsafe") + // Open opens or creates a new SQLite database at the given path with FTS5 and WAL mode enabled. func Open(path string) (*Database, error) { + d, err := openWithoutSetup(path) + if err != nil { + return nil, err + } + if err := d.SetupSchema(); err != nil { + _ = d.Close() + return nil, err + } + return d, nil +} + +func openWithoutSetup(path string) (*Database, error) { + return openWriteConnection(path, false) +} + +func openMigrationWithoutSetup(path string) (*Database, error) { + return openWriteConnection(path, true) +} + +func openWriteConnection(path string, migrationImmediate bool) (*Database, error) { + canonicalPath, err := canonicalizeDBPath(path) + if err != nil { + return nil, err + } // modernc.org/sqlite honors the `_pragma=name(value)` DSN syntax; the mattn-style // `_name=value` form is silently ignored (leaving rollback journal mode + no busy timeout). - db, err := sql.Open("sqlite", path+"?_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL)&_pragma=busy_timeout(5000)") + dsn := canonicalPath + "?_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL)&_pragma=busy_timeout(5000)" + if migrationImmediate { + dsn += "&_txlock=immediate" + } + db, err := sql.Open("sqlite", dsn) if err != nil { - return nil, fmt.Errorf("opening database %s: %w", path, err) + return nil, fmt.Errorf("opening database %s: %w", canonicalPath, err) } // Test the connection if err := db.Ping(); err != nil { _ = db.Close() - return nil, fmt.Errorf("ping database %s: %w", path, err) + return nil, fmt.Errorf("ping database %s: %w", canonicalPath, err) } // Enable FK enforcement (required for ON DELETE CASCADE in V2 schema) @@ -37,13 +77,48 @@ func Open(path string) (*Database, error) { return nil, fmt.Errorf("enable foreign keys: %w", err) } - d := &Database{db: db} - if err := d.SetupSchema(); err != nil { - _ = db.Close() - return nil, err + return &Database{db: db, path: canonicalPath}, nil +} + +func OpenCompatible(ctx context.Context, path string) (*Database, *compat.Diagnostic, error) { + inspect, err := OpenReadOnly(path) + if errors.Is(err, fs.ErrNotExist) { + db, openErr := Open(path) + return db, nil, openErr + } + if err != nil { + return nil, nil, err + } + canonicalPath := inspect.path + plan, diag, err := compat.InspectIndex(ctx, inspect.DB()) + closeErr := inspect.Close() + if err != nil || diag != nil { + return nil, diag, err + } + if closeErr != nil { + return nil, nil, closeErr + } + if len(plan.Steps) == 0 { + db, openErr := openWithoutSetup(canonicalPath) + return db, nil, openErr } - return d, nil + migrationDB, err := openMigrationWithoutSetup(canonicalPath) + if err != nil { + return nil, nil, err + } + if err := openCompatibleApplyMigrationPlan(migrationDB, ctx, plan); err != nil { + _ = migrationDB.Close() + return nil, nil, err + } + if err := migrationDB.Close(); err != nil { + return nil, nil, err + } + db, err := openWithoutSetup(canonicalPath) + if err != nil { + return nil, nil, err + } + return db, nil, nil } // OpenReadOnly opens an existing SQLite database in read-only mode. @@ -51,23 +126,84 @@ func Open(path string) (*Database, error) { func OpenReadOnly(path string) (*Database, error) { // Fail fast if DB file doesn't exist if _, err := os.Stat(path); os.IsNotExist(err) { - return nil, fmt.Errorf("backscroll database not found: %s", path) + return nil, fmt.Errorf("backscroll database not found: %s: %w", path, fs.ErrNotExist) + } + canonicalPath, err := canonicalizeExistingDBPath(path) + if err != nil { + return nil, err } // Journal mode is persisted in the DB file (set by the write connection); a read-only // connection only needs the busy timeout so queries wait out a concurrent writer's lock. - db, err := sql.Open("sqlite", "file:"+path+"?mode=ro&_pragma=busy_timeout(5000)") + db, err := sql.Open("sqlite", "file:"+canonicalPath+"?mode=ro&_pragma=busy_timeout(5000)") if err != nil { - return nil, fmt.Errorf("opening readonly database %s: %w", path, err) + return nil, fmt.Errorf("opening readonly database %s: %w", canonicalPath, err) } // Test the connection if err := db.Ping(); err != nil { _ = db.Close() - return nil, fmt.Errorf("ping readonly database %s: %w", path, err) + return nil, fmt.Errorf("ping readonly database %s: %w", canonicalPath, err) + } + + return &Database{db: db, path: canonicalPath}, nil +} + +// OpenImmutableReadOnly opens an existing SQLite database without creating or +// touching SQLite sidecar files. It refuses non-empty WAL files because an +// immutable view can miss committed frames that are not checkpointed into the +// main database file. +func OpenImmutableReadOnly(path string) (*Database, error) { + if _, err := os.Stat(path); os.IsNotExist(err) { + return nil, fmt.Errorf("backscroll database not found: %s: %w", path, fs.ErrNotExist) + } + canonicalPath, err := canonicalizeExistingDBPath(path) + if err != nil { + return nil, err + } + walPath := canonicalPath + "-wal" + if wal, err := os.Stat(walPath); err == nil { + if wal.Size() > 0 { + return nil, fmt.Errorf("%w: %s", ErrImmutableReadOnlyWALUnsafe, canonicalPath) + } + } else if !os.IsNotExist(err) { + return nil, fmt.Errorf("stat WAL sidecar %s: %w", walPath, err) + } + + db, err := sql.Open("sqlite", "file:"+canonicalPath+"?mode=ro&immutable=1&_pragma=busy_timeout(5000)") + if err != nil { + return nil, fmt.Errorf("opening immutable readonly database %s: %w", canonicalPath, err) + } + if err := db.Ping(); err != nil { + _ = db.Close() + return nil, fmt.Errorf("ping immutable readonly database %s: %w", canonicalPath, err) + } + return &Database{db: db, path: canonicalPath}, nil +} + +func canonicalizeDBPath(path string) (string, error) { + abs, err := filepath.Abs(path) + if err != nil { + return "", fmt.Errorf("canonicalize database path %s: %w", path, err) } + if _, err := os.Stat(abs); err == nil { + return canonicalizeExistingDBPath(abs) + } else if !os.IsNotExist(err) { + return "", fmt.Errorf("stat database path %s: %w", abs, err) + } + return abs, nil +} - return &Database{db: db}, nil +func canonicalizeExistingDBPath(path string) (string, error) { + abs, err := filepath.Abs(path) + if err != nil { + return "", fmt.Errorf("canonicalize database path %s: %w", path, err) + } + realPath, err := filepath.EvalSymlinks(abs) + if err != nil { + return "", fmt.Errorf("resolve database path %s: %w", abs, err) + } + return realPath, nil } // Close closes the database connection. diff --git a/internal/storage/storage_test.go b/internal/storage/storage_test.go index 74032bc..df794c0 100644 --- a/internal/storage/storage_test.go +++ b/internal/storage/storage_test.go @@ -1,9 +1,11 @@ package storage import ( + "errors" "fmt" "os" "path/filepath" + "reflect" "testing" "time" @@ -158,6 +160,75 @@ func TestOpenReadOnlyHasBusyTimeout(t *testing.T) { } } +func TestOpenReadOnlyLiveWALSeesCommittedRows(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "live_wal.db") + writer := openTestDBWithLiveWAL(t, dbPath) + defer func() { _ = writer.Close() }() + + readonly, err := OpenReadOnly(dbPath) + if err != nil { + t.Fatalf("OpenReadOnly with live WAL: %v", err) + } + defer func() { _ = readonly.Close() }() + + var count int + if err := readonly.db.QueryRow("SELECT COUNT(*) FROM search_items WHERE text = 'live WAL committed row'").Scan(&count); err != nil { + t.Fatalf("query live WAL row through OpenReadOnly: %v", err) + } + if count != 1 { + t.Fatalf("live WAL row count = %d, want 1", count) + } +} + +func TestOpenImmutableReadOnlyRefusesLiveWALWithoutSidecarMutation(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "immutable_live_wal.db") + writer := openTestDBWithLiveWAL(t, dbPath) + defer func() { _ = writer.Close() }() + before := snapshotDBSidecars(t, dbPath) + + immutable, err := OpenImmutableReadOnly(dbPath) + if err == nil { + _ = immutable.Close() + t.Fatal("OpenImmutableReadOnly with live WAL succeeded; want unsafe WAL error") + } + if !errors.Is(err, ErrImmutableReadOnlyWALUnsafe) { + t.Fatalf("OpenImmutableReadOnly error = %v, want ErrImmutableReadOnlyWALUnsafe", err) + } + + after := snapshotDBSidecars(t, dbPath) + if !reflect.DeepEqual(after, before) { + t.Fatalf("OpenImmutableReadOnly mutated sidecars\nbefore=%s\nafter= %s", describeStorageSidecars(before), describeStorageSidecars(after)) + } +} + +func TestOpenImmutableReadOnlyDoesNotCreateSidecars(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "immutable_clean.db") + writer := openTestDBWithLiveWAL(t, dbPath) + if _, err := writer.db.Exec("PRAGMA wal_checkpoint(FULL)"); err != nil { + t.Fatalf("checkpoint clean immutable fixture: %v", err) + } + if err := writer.Close(); err != nil { + t.Fatalf("close clean immutable fixture: %v", err) + } + _ = os.Remove(dbPath + "-wal") + _ = os.Remove(dbPath + "-shm") + + immutable, err := OpenImmutableReadOnly(dbPath) + if err != nil { + t.Fatalf("OpenImmutableReadOnly clean database: %v", err) + } + defer func() { _ = immutable.Close() }() + var count int + if err := immutable.db.QueryRow("SELECT COUNT(*) FROM search_items WHERE text = 'live WAL committed row'").Scan(&count); err != nil { + t.Fatalf("query immutable clean database: %v", err) + } + if count != 1 { + t.Fatalf("immutable row count = %d, want 1", count) + } + assertStorageSidecarMissing(t, dbPath+"-wal") + assertStorageSidecarMissing(t, dbPath+"-shm") +} + // TestOpenReadOnlyNonExistent verifies that opening a non-existent database fails. func TestOpenReadOnlyNonExistent(t *testing.T) { _, err := OpenReadOnly("/nonexistent/path/db.sqlite") @@ -166,6 +237,87 @@ func TestOpenReadOnlyNonExistent(t *testing.T) { } } +func openTestDBWithLiveWAL(t *testing.T, dbPath string) *Database { + t.Helper() + db, err := Open(dbPath) + if err != nil { + t.Fatalf("Open live WAL fixture: %v", err) + } + if err := db.SyncFiles([]IndexedFile{{ + SourcePath: "/sessions/live-wal.jsonl", + Source: "session", + Hash: "live-wal-hash", + Project: "project", + Messages: []IndexedMessage{{ + Ordinal: 0, + Role: "user", + Text: "live WAL committed row", + UUID: "33333333-3333-4333-8333-333333333333", + Timestamp: "2026-08-18T00:00:00Z", + ContentType: "text", + }}, + }}); err != nil { + _ = db.Close() + t.Fatalf("seed live WAL fixture: %v", err) + } + wal, err := os.Stat(dbPath + "-wal") + if err != nil { + _ = db.Close() + t.Fatalf("stat live WAL sidecar: %v", err) + } + if wal.Size() == 0 { + _ = db.Close() + t.Fatal("live WAL sidecar is empty; test fixture did not create committed WAL frames") + } + return db +} + +type storageSidecarSnapshot struct { + Data []byte + Mode os.FileMode + MTime time.Time +} + +func snapshotDBSidecars(t *testing.T, dbPath string) map[string]storageSidecarSnapshot { + t.Helper() + result := map[string]storageSidecarSnapshot{} + for _, suffix := range []string{"-wal", "-shm"} { + path := dbPath + suffix + info, err := os.Stat(path) + if os.IsNotExist(err) { + continue + } + if err != nil { + t.Fatalf("stat sidecar %s: %v", path, err) + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read sidecar %s: %v", path, err) + } + result[suffix] = storageSidecarSnapshot{Data: data, Mode: info.Mode(), MTime: info.ModTime()} + } + return result +} + +func describeStorageSidecars(snapshot map[string]storageSidecarSnapshot) string { + parts := make([]string, 0, len(snapshot)) + for _, suffix := range []string{"-wal", "-shm"} { + entry, ok := snapshot[suffix] + if !ok { + continue + } + parts = append(parts, fmt.Sprintf("%s bytes=%d mode=%s mtime=%s", suffix, len(entry.Data), entry.Mode, entry.MTime.Format(time.RFC3339Nano))) + } + return fmt.Sprintf("%v", parts) +} + +func assertStorageSidecarMissing(t *testing.T, path string) { + t.Helper() + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Fatalf("sidecar %s exists after immutable read-only open (err=%v)", path, err) + } +} + // TestSyncFiles inserts and updates files. func TestSyncFiles(t *testing.T) { db, cleanup := newTestDB(t) diff --git a/internal/storage/unit_test.go b/internal/storage/unit_test.go index 0d9b1fa..776538c 100644 --- a/internal/storage/unit_test.go +++ b/internal/storage/unit_test.go @@ -631,6 +631,101 @@ func TestQueryIndexedRecords(t *testing.T) { } } +func TestQueryIndexedRecordsReturnsCanonicalModel(t *testing.T) { + db, cleanup := newTestDB(t) + defer cleanup() + + project := "canonical-project" + uuid := getTestUUID() + timestamp := "2026-08-18T12:34:56Z" + if err := db.SyncFiles([]IndexedFile{ + { + SourcePath: "/sentinel/canonical.md", + Source: "decision", + Hash: "canonical-hash", + Project: project, + Messages: []IndexedMessage{ + { + Ordinal: 7, + Role: "assistant", + Text: "canonical model text sentinel", + UUID: uuid, + Timestamp: timestamp, + ContentType: "text/markdown", + }, + }, + }, + }); err != nil { + t.Fatalf("SyncFiles: %v", err) + } + + path := "/sentinel/canonical.md" + records, err := db.QueryIndexedRecords(IndexedRecordQuery{SourcePath: &path}) + if err != nil { + t.Fatalf("QueryIndexedRecords: %v", err) + } + var got []models.IndexedRecord = records + if len(got) != 1 { + t.Fatalf("expected 1 canonical record, got %d", len(got)) + } + record := got[0] + if record.Source != "decision" { + t.Errorf("Source = %q, want %q", record.Source, "decision") + } + if record.SourcePath != path { + t.Errorf("SourcePath = %q, want %q", record.SourcePath, path) + } + if record.Ordinal != 7 { + t.Errorf("Ordinal = %d, want 7", record.Ordinal) + } + if record.Role != "assistant" { + t.Errorf("Role = %q, want %q", record.Role, "assistant") + } + if record.Text != "canonical model text sentinel" { + t.Errorf("Text = %q, want sentinel text", record.Text) + } + if record.Project == nil || *record.Project != project { + t.Errorf("Project = %v, want %q", record.Project, project) + } + if record.UUID == nil || *record.UUID != uuid { + t.Errorf("UUID = %v, want %q", record.UUID, uuid) + } + if record.Timestamp == nil || *record.Timestamp != timestamp { + t.Errorf("Timestamp = %v, want %q", record.Timestamp, timestamp) + } + if record.ContentType != "text/markdown" { + t.Errorf("ContentType = %q, want %q", record.ContentType, "text/markdown") + } + + if _, err := db.DB().Exec(` + INSERT INTO search_items (source, source_path, ordinal, role, text) + VALUES (?, ?, ?, ?, ?) + `, "session", "/sentinel/nulls.jsonl", 8, "user", "null pointer sentinel"); err != nil { + t.Fatalf("insert null sentinel row: %v", err) + } + nullPath := "/sentinel/nulls.jsonl" + nullRecords, err := db.QueryIndexedRecords(IndexedRecordQuery{SourcePath: &nullPath}) + if err != nil { + t.Fatalf("QueryIndexedRecords null sentinel: %v", err) + } + var nullGot []models.IndexedRecord = nullRecords + if len(nullGot) != 1 { + t.Fatalf("expected 1 null sentinel record, got %d", len(nullGot)) + } + if nullGot[0].Project != nil { + t.Errorf("Project = %v, want nil", nullGot[0].Project) + } + if nullGot[0].UUID != nil { + t.Errorf("UUID = %v, want nil", nullGot[0].UUID) + } + if nullGot[0].Timestamp != nil { + t.Errorf("Timestamp = %v, want nil", nullGot[0].Timestamp) + } + if nullGot[0].ContentType != "text" { + t.Errorf("ContentType = %q, want default text", nullGot[0].ContentType) + } +} + func TestPurgeWithISODateFormat(t *testing.T) { db, cleanup := newTestDB(t) defer cleanup() diff --git a/scripts/calibration-extract/main.go b/scripts/calibration-extract/main.go index 41302c5..db7aa48 100644 --- a/scripts/calibration-extract/main.go +++ b/scripts/calibration-extract/main.go @@ -1,7 +1,9 @@ package main import ( + "context" "encoding/csv" + "errors" "flag" "fmt" "log" @@ -10,6 +12,7 @@ import ( "path/filepath" "sort" + "github.com/pablontiv/backscroll/internal/compat" "github.com/pablontiv/backscroll/internal/storage" ) @@ -41,11 +44,10 @@ func main() { log.Fatalf("get current user: %v", err) } dbPath := filepath.Join(currentUser.HomeDir, ".backscroll.db") - db, err := storage.OpenReadOnly(dbPath) + db, err := openInspectedCalibrationDatabase(context.Background(), dbPath) if err != nil { log.Fatalf("open database: %v", err) } - defer func() { _ = db.Close() }() // Query corrections with min_confidence=0.4 opts := storage.CorrectionAggOpts{ @@ -60,6 +62,10 @@ func main() { // Stratified sampling with window context population samples := stratifyWithDB(db, candidates, *total, *perDetector, *perSession) + if err := db.Close(); err != nil { + log.Fatalf("close database: %v", err) + } + // Output CSV if err := writeCSV(*output, samples); err != nil { log.Fatalf("write csv: %v", err) @@ -68,6 +74,34 @@ func main() { fmt.Printf("Extracted %d samples to %s\n", len(samples), *output) } +func openInspectedCalibrationDatabase(ctx context.Context, dbPath string) (*storage.Database, error) { + db, err := storage.OpenReadOnly(dbPath) + if err != nil { + return nil, err + } + plan, diag, err := compat.InspectIndex(ctx, db.DB()) + if err != nil { + return nil, closeCalibrationDB(db, fmt.Errorf("inspect index: %w", err)) + } + if diag != nil { + return nil, closeCalibrationDB(db, fmt.Errorf("%s: %s", diag.Code, diag.Summary)) + } + if len(plan.Steps) > 0 { + return nil, closeCalibrationDB(db, fmt.Errorf("%s: index schema %s has %d pending migration step(s)", compat.CodeIndexStale, plan.From.Signature, len(plan.Steps))) + } + return db, nil +} + +func closeCalibrationDB(db *storage.Database, err error) error { + if db == nil { + return err + } + if closeErr := db.Close(); closeErr != nil { + return errors.Join(err, fmt.Errorf("close database: %w", closeErr)) + } + return err +} + // Sample represents one row in the output CSV. type Sample struct { UUID string diff --git a/scripts/calibration-extract/main_test.go b/scripts/calibration-extract/main_test.go index 79c4aaa..a074f73 100644 --- a/scripts/calibration-extract/main_test.go +++ b/scripts/calibration-extract/main_test.go @@ -1,10 +1,13 @@ package main import ( + "context" "os" "path/filepath" + "strings" "testing" + "github.com/pablontiv/backscroll/internal/compat" "github.com/pablontiv/backscroll/internal/storage" ) @@ -79,6 +82,40 @@ func createTestCorrectionSignals(t *testing.T, db *storage.Database, signals []s } } +func TestCalibrationExtractRejectsUnsupportedIndex(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "unsupported.db") + db, err := storage.Open(dbPath) + if err != nil { + t.Fatalf("open db: %v", err) + } + if _, err := db.DB().Exec(`CREATE TABLE unexpected_shape_marker (id INTEGER PRIMARY KEY)`); err != nil { + _ = db.Close() + t.Fatalf("make unsupported: %v", err) + } + if _, err := db.DB().Exec(`PRAGMA wal_checkpoint(TRUNCATE)`); err != nil { + _ = db.Close() + t.Fatalf("checkpoint: %v", err) + } + if err := db.Close(); err != nil { + t.Fatalf("close db: %v", err) + } + + got, err := openInspectedCalibrationDatabase(context.Background(), dbPath) + if err == nil { + if got != nil { + _ = got.Close() + } + t.Fatal("openInspectedCalibrationDatabase succeeded for unsupported index") + } + if got != nil { + _ = got.Close() + t.Fatal("openInspectedCalibrationDatabase returned a db with an error") + } + if !strings.Contains(err.Error(), string(compat.CodeUnsupportedLineage)) { + t.Fatalf("error %q does not include diagnostic code %q", err, compat.CodeUnsupportedLineage) + } +} + func TestStratifyPerDetectorEqualQuotas(t *testing.T) { // Test equal quota distribution: 12 lexicon + 12 denial candidates // --total 20 --per-detector 10 → stratify distributes ~10 per detector diff --git a/tests/fixtures/recovery/active-v13.sql b/tests/fixtures/recovery/active-v13.sql new file mode 100644 index 0000000..0abd8fd --- /dev/null +++ b/tests/fixtures/recovery/active-v13.sql @@ -0,0 +1,210 @@ +-- Backscroll release schema fixture: v13.sql +-- Hermetic schema-only fixture captured for compatibility tests. + +BEGIN TRANSACTION; +CREATE TABLE IF NOT EXISTS schema_migrations ( + version INTEGER PRIMARY KEY, + name TEXT NOT NULL, + applied_on TEXT NOT NULL, + checksum TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS indexed_files ( + path TEXT PRIMARY KEY, + hash TEXT NOT NULL, + last_indexed DATETIME DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS search_items ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source TEXT NOT NULL DEFAULT 'session', + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + role TEXT NOT NULL, + text TEXT NOT NULL, + timestamp TEXT, + uuid TEXT UNIQUE, + project TEXT, + content_type TEXT NOT NULL DEFAULT 'text', + extraction_version INTEGER, + was_interrupted INTEGER +); + +CREATE INDEX IF NOT EXISTS idx_search_items_source_path ON search_items(source_path); +CREATE INDEX IF NOT EXISTS idx_search_items_project ON search_items(project); + +CREATE TABLE IF NOT EXISTS session_tags ( + source_path TEXT NOT NULL, + tag TEXT NOT NULL, + PRIMARY KEY (source_path, tag) +); + +CREATE INDEX IF NOT EXISTS idx_session_tags_tag ON session_tags(tag); + +CREATE TABLE IF NOT EXISTS dynamic_stopwords (term TEXT PRIMARY KEY); + +CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5( + text, + content=search_items, + content_rowid=id, + tokenize='porter unicode61' +); + +CREATE VIRTUAL TABLE IF NOT EXISTS messages_vocab USING fts5vocab(messages_fts, 'row'); + +CREATE VIRTUAL TABLE IF NOT EXISTS tool_fts USING fts5( + text, + content=search_items, + content_rowid=id, + tokenize='trigram' +); + +CREATE VIRTUAL TABLE IF NOT EXISTS tool_vocab USING fts5vocab(tool_fts, 'row'); + +CREATE TRIGGER IF NOT EXISTS search_items_ai_tool AFTER INSERT ON search_items +WHEN new.content_type = 'tool' BEGIN + INSERT INTO tool_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_ai_msg AFTER INSERT ON search_items +WHEN new.content_type IN ('text', 'code', 'reasoning') BEGIN + INSERT INTO messages_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_ad_tool AFTER DELETE ON search_items +WHEN old.content_type = 'tool' BEGIN + INSERT INTO tool_fts(tool_fts, rowid, text) VALUES('delete', old.id, old.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_ad_msg AFTER DELETE ON search_items +WHEN old.content_type IN ('text', 'code', 'reasoning') BEGIN + INSERT INTO messages_fts(messages_fts, rowid, text) VALUES('delete', old.id, old.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_au_tool AFTER UPDATE ON search_items +WHEN old.content_type = 'tool' BEGIN + INSERT INTO tool_fts(tool_fts, rowid, text) VALUES('delete', old.id, old.text); + INSERT INTO tool_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_au_msg AFTER UPDATE ON search_items +WHEN old.content_type IN ('text', 'code', 'reasoning') BEGIN + INSERT INTO messages_fts(messages_fts, rowid, text) VALUES('delete', old.id, old.text); + INSERT INTO messages_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TABLE IF NOT EXISTS chunks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source_id TEXT NOT NULL, + chunk_idx INTEGER NOT NULL, + content TEXT NOT NULL, + token_count INTEGER NOT NULL, + created_at INTEGER NOT NULL, + embedding BLOB, + UNIQUE(source_id, chunk_idx) +); + +CREATE INDEX IF NOT EXISTS idx_chunks_source_id ON chunks (source_id); + +CREATE TABLE IF NOT EXISTS embedding_metadata ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + chunk_id INTEGER NOT NULL REFERENCES chunks(id) ON DELETE CASCADE, + model_name TEXT NOT NULL, + model_version TEXT NOT NULL, + dimensions INTEGER NOT NULL, + created_at INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS tool_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + message_uuid TEXT, + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + tool_name TEXT NOT NULL, + command_head TEXT, + is_error INTEGER, + exit_code INTEGER, + extraction_version INTEGER NOT NULL, + UNIQUE(source_path, ordinal) +); +CREATE INDEX IF NOT EXISTS idx_tool_events_tool ON tool_events(tool_name); +CREATE INDEX IF NOT EXISTS idx_tool_events_uuid ON tool_events(message_uuid); +CREATE UNIQUE INDEX IF NOT EXISTS idx_tool_events_uuid_unique ON tool_events(message_uuid) WHERE message_uuid IS NOT NULL; + +CREATE TABLE IF NOT EXISTS message_templates ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + signature TEXT UNIQUE NOT NULL, + normalization_version INTEGER NOT NULL, + template_text TEXT NOT NULL, + occurrence_count INTEGER NOT NULL DEFAULT 1, + first_seen TEXT, + last_seen TEXT +); +CREATE INDEX IF NOT EXISTS idx_templates_sig ON message_templates(signature); +CREATE INDEX IF NOT EXISTS idx_templates_version ON message_templates(normalization_version); + +CREATE TABLE IF NOT EXISTS template_matches ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + template_id INTEGER NOT NULL, + item_uuid TEXT, + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + UNIQUE(source_path, ordinal, template_id), + FOREIGN KEY(template_id) REFERENCES message_templates(id) +); +CREATE INDEX IF NOT EXISTS idx_matches_template ON template_matches(template_id); +CREATE INDEX IF NOT EXISTS idx_matches_uuid ON template_matches(item_uuid); + +CREATE TABLE IF NOT EXISTS correction_signals ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + item_uuid TEXT, + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + detector TEXT NOT NULL, + confidence REAL NOT NULL, + extraction_version INTEGER NOT NULL, + UNIQUE(source_path, ordinal, detector) +); +CREATE INDEX IF NOT EXISTS idx_correction_signals_detector ON correction_signals(detector); +CREATE INDEX IF NOT EXISTS idx_correction_signals_confidence ON correction_signals(confidence DESC); + +CREATE TABLE IF NOT EXISTS annotations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + item_uuid TEXT, + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + kind TEXT NOT NULL, + label TEXT NOT NULL, + source TEXT NOT NULL DEFAULT 'agent', + created_at TEXT NOT NULL, + UNIQUE(source_path, ordinal, kind) +); +CREATE INDEX IF NOT EXISTS idx_annotations_uuid ON annotations(item_uuid); +CREATE INDEX IF NOT EXISTS idx_annotations_kind ON annotations(kind); + +CREATE INDEX IF NOT EXISTS idx_template_matches_source ON template_matches(source_path); +CREATE INDEX IF NOT EXISTS idx_correction_signals_source ON correction_signals(source_path); + +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (1, 'V1 core schema', '1970-01-01 00:00:00', '4e07949ccd3912fb3c0e149be9a2e05fdd51f8cedb8df1f28b3bb5ac5afe532a'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (2, 'V2 embedding tables', '1970-01-01 00:00:00', '37dc9627f01f0e2d0fbea6bba5cd9f609d5da05089eeb9541a057dd2290cf8af'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (3, 'V3 embedding blob column', '1970-01-01 00:00:00', '36cd183f10ff84ab4753be027078cdd710b46efdc830053d4a641af725006ea5'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (4, 'V4 tool_fts trigram index', '1970-01-01 00:00:00', '77e59cf515f33c466282e0f3e921377f584794d0b7cade1704f566374a569a55'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (5, 'V5 drop phantom session_events', '1970-01-01 00:00:00', 'aedaab81efb6bc34d3f664468b71c1a716f5b55d5132156cb43bd7462d549c7b'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (6, 'V6 drop phantom source_metadata column', '1970-01-01 00:00:00', 'a327b9b6e7b8f5fe369c9fc08093ac80a87640daa515890af94b269d430e9378'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (7, 'V7 reasoning content_type routes to messages_fts', '1970-01-01 00:00:00', 'a80704442c2a0084f98e4bc53978364b14d1c6bd3bd99ba6119a2a9ecbd685e7'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (8, 'V8 perennity: extraction_version, was_interrupted, tool_events', '1970-01-01 00:00:00', '6853d72ded3bdc775b52507321c31df44bf35e8277719432ca8aa126ee16cec1'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (9, 'V9 tool_events uuid uniqueness index', '1970-01-01 00:00:00', 'b16094805a4e08f6e0dd56bce5266c7c5fd71934389da9d13c4132076e546ca2'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (10, 'V10 template mining: message_templates, template_matches', '1970-01-01 00:00:00', '0e548d0cb6c47147726f943bfc860500ca9a9bc821df0601998876ea5e9652c2'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (11, 'V11 correction detection: correction_signals', '1970-01-01 00:00:00', '5a7180f901c5feacb67cd104a61e7b7ba9cec69aaeab1d2b5d723459dc590456'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (12, 'V12 agent classification: annotations (free-form labels; enum freeze deferred)', '1970-01-01 00:00:00', 'b3fb66fd2924a9f07a3e4ec0ba253fc1d6965664615a795be6b22d01ab292108'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (13, 'V13 backfill discovery indexes', '1970-01-01 00:00:00', '2172ce531c670806933ffe3005fdc0a2ebb8eb3f84d2bd0d8fa608dddb5d136e'); + + +-- Recovery adapter payload sentinels. +INSERT INTO indexed_files (path, hash, last_indexed) VALUES ('/fixtures/recovery/active-v13.jsonl', 'hash-active-v13', '2026-08-18T00:00:00Z'); +INSERT INTO indexed_files (path, hash, last_indexed) VALUES ('/fixtures/recovery/active-v13-defaults.jsonl', 'hash-active-v13-defaults', '2026-08-18T00:00:01Z'); +INSERT INTO search_items (source, source_path, ordinal, role, text, timestamp, uuid, project, content_type) +VALUES ('source-active-v13', '/fixtures/recovery/active-v13.jsonl', 41, 'assistant', 'text sentinel for active-v13', '2026-08-18T12:34:56Z', 'uuid-active-v13', 'project-active-v13', 'content/active-v13'); +INSERT INTO search_items (source, source_path, ordinal, role, text) +VALUES ('session', '/fixtures/recovery/active-v13-defaults.jsonl', 42, 'user', 'default sentinel for active-v13'); +COMMIT; diff --git a/tests/fixtures/recovery/stranded-v3-no-source-metadata.sql b/tests/fixtures/recovery/stranded-v3-no-source-metadata.sql new file mode 100644 index 0000000..deb738b --- /dev/null +++ b/tests/fixtures/recovery/stranded-v3-no-source-metadata.sql @@ -0,0 +1,127 @@ +-- Backscroll release schema fixture: v3-no-source-metadata.sql +-- Hermetic schema-only fixture captured for compatibility tests. +-- No manifest release tag maps to this fixture. +-- It preserves a partially migrated V3 shape for compatibility triangulation +-- only: source_metadata is absent from search_items even though no published +-- release in the manifest shipped this exact schema_migrations lineage. + +BEGIN TRANSACTION; +CREATE TABLE IF NOT EXISTS schema_migrations ( + version INTEGER PRIMARY KEY, + name TEXT NOT NULL, + applied_on TEXT NOT NULL, + checksum TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS indexed_files ( + path TEXT PRIMARY KEY, + hash TEXT NOT NULL, + last_indexed DATETIME DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS search_items ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source TEXT NOT NULL DEFAULT 'session', + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + role TEXT NOT NULL, + text TEXT NOT NULL, + timestamp TEXT, + uuid TEXT UNIQUE, + project TEXT, + content_type TEXT NOT NULL DEFAULT 'text' +); + +CREATE INDEX IF NOT EXISTS idx_search_items_source_path ON search_items(source_path); +CREATE INDEX IF NOT EXISTS idx_search_items_project ON search_items(project); + +CREATE TABLE IF NOT EXISTS session_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + schema_version INTEGER NOT NULL DEFAULT 1, + source TEXT NOT NULL DEFAULT 'session', + source_path TEXT NOT NULL, + project TEXT, + ordinal INTEGER NOT NULL, + timestamp TEXT, + event_type TEXT NOT NULL, + actor TEXT, + role TEXT, + tool_name TEXT, + tool_id TEXT, + command TEXT, + cwd TEXT, + exit_code INTEGER, + is_error INTEGER, + snippet TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_session_events_order ON session_events(source_path, ordinal, timestamp, id); +CREATE INDEX IF NOT EXISTS idx_session_events_project ON session_events(project); + +CREATE TABLE IF NOT EXISTS session_tags ( + source_path TEXT NOT NULL, + tag TEXT NOT NULL, + PRIMARY KEY (source_path, tag) +); + +CREATE INDEX IF NOT EXISTS idx_session_tags_tag ON session_tags(tag); + +CREATE TABLE IF NOT EXISTS dynamic_stopwords (term TEXT PRIMARY KEY); + +CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5( + text, + content=search_items, + content_rowid=id, + tokenize='porter unicode61' +); + +CREATE VIRTUAL TABLE IF NOT EXISTS messages_vocab USING fts5vocab(messages_fts, 'row'); + +CREATE TRIGGER IF NOT EXISTS search_items_ai AFTER INSERT ON search_items BEGIN + INSERT INTO messages_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_ad AFTER DELETE ON search_items BEGIN + INSERT INTO messages_fts(messages_fts, rowid, text) VALUES('delete', old.id, old.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_au AFTER UPDATE ON search_items BEGIN + INSERT INTO messages_fts(messages_fts, rowid, text) VALUES('delete', old.id, old.text); + INSERT INTO messages_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TABLE IF NOT EXISTS chunks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source_id TEXT NOT NULL, + chunk_idx INTEGER NOT NULL, + content TEXT NOT NULL, + token_count INTEGER NOT NULL, + created_at INTEGER NOT NULL, + embedding BLOB, + UNIQUE(source_id, chunk_idx) +); + +CREATE INDEX IF NOT EXISTS idx_chunks_source_id ON chunks (source_id); + +CREATE TABLE IF NOT EXISTS embedding_metadata ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + chunk_id INTEGER NOT NULL REFERENCES chunks(id) ON DELETE CASCADE, + model_name TEXT NOT NULL, + model_version TEXT NOT NULL, + dimensions INTEGER NOT NULL, + created_at INTEGER NOT NULL +); + +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (1, 'V1 core schema', '1970-01-01 00:00:00', '4e07949ccd3912fb3c0e149be9a2e05fdd51f8cedb8df1f28b3bb5ac5afe532a'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (2, 'V2 embedding tables', '1970-01-01 00:00:00', '37dc9627f01f0e2d0fbea6bba5cd9f609d5da05089eeb9541a057dd2290cf8af'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (3, 'V3 embedding blob column', '1970-01-01 00:00:00', '36cd183f10ff84ab4753be027078cdd710b46efdc830053d4a641af725006ea5'); + + +-- Recovery adapter payload sentinels. +INSERT INTO indexed_files (path, hash, last_indexed) VALUES ('/fixtures/recovery/stranded-v3-no-source-metadata.jsonl', 'hash-stranded-v3-no-source-metadata', '2026-08-18T00:00:00Z'); +INSERT INTO indexed_files (path, hash, last_indexed) VALUES ('/fixtures/recovery/stranded-v3-no-source-metadata-defaults.jsonl', 'hash-stranded-v3-no-source-metadata-defaults', '2026-08-18T00:00:01Z'); +INSERT INTO search_items (source, source_path, ordinal, role, text, timestamp, uuid, project, content_type) +VALUES ('source-stranded-v3-no-source-metadata', '/fixtures/recovery/stranded-v3-no-source-metadata.jsonl', 41, 'assistant', 'text sentinel for stranded-v3-no-source-metadata', '2026-08-18T12:34:56Z', 'uuid-stranded-v3-no-source-metadata', 'project-stranded-v3-no-source-metadata', 'content/stranded-v3-no-source-metadata'); +INSERT INTO search_items (source, source_path, ordinal, role, text) +VALUES ('session', '/fixtures/recovery/stranded-v3-no-source-metadata-defaults.jsonl', 42, 'user', 'default sentinel for stranded-v3-no-source-metadata'); +COMMIT; diff --git a/tests/fixtures/recovery/stranded-v7.sql b/tests/fixtures/recovery/stranded-v7.sql new file mode 100644 index 0000000..b056bef --- /dev/null +++ b/tests/fixtures/recovery/stranded-v7.sql @@ -0,0 +1,132 @@ +-- Backscroll release schema fixture: v7.sql +-- Hermetic schema-only fixture captured for compatibility tests. + +BEGIN TRANSACTION; +CREATE TABLE IF NOT EXISTS schema_migrations ( + version INTEGER PRIMARY KEY, + name TEXT NOT NULL, + applied_on TEXT NOT NULL, + checksum TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS indexed_files ( + path TEXT PRIMARY KEY, + hash TEXT NOT NULL, + last_indexed DATETIME DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS search_items ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source TEXT NOT NULL DEFAULT 'session', + source_path TEXT NOT NULL, + ordinal INTEGER NOT NULL, + role TEXT NOT NULL, + text TEXT NOT NULL, + timestamp TEXT, + uuid TEXT UNIQUE, + project TEXT, + content_type TEXT NOT NULL DEFAULT 'text' +); + +CREATE INDEX IF NOT EXISTS idx_search_items_source_path ON search_items(source_path); +CREATE INDEX IF NOT EXISTS idx_search_items_project ON search_items(project); + +CREATE TABLE IF NOT EXISTS session_tags ( + source_path TEXT NOT NULL, + tag TEXT NOT NULL, + PRIMARY KEY (source_path, tag) +); + +CREATE INDEX IF NOT EXISTS idx_session_tags_tag ON session_tags(tag); + +CREATE TABLE IF NOT EXISTS dynamic_stopwords (term TEXT PRIMARY KEY); + +CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5( + text, + content=search_items, + content_rowid=id, + tokenize='porter unicode61' +); + +CREATE VIRTUAL TABLE IF NOT EXISTS messages_vocab USING fts5vocab(messages_fts, 'row'); + +CREATE VIRTUAL TABLE IF NOT EXISTS tool_fts USING fts5( + text, + content=search_items, + content_rowid=id, + tokenize='trigram' +); + +CREATE VIRTUAL TABLE IF NOT EXISTS tool_vocab USING fts5vocab(tool_fts, 'row'); + +CREATE TRIGGER IF NOT EXISTS search_items_ai_tool AFTER INSERT ON search_items +WHEN new.content_type = 'tool' BEGIN + INSERT INTO tool_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_ai_msg AFTER INSERT ON search_items +WHEN new.content_type IN ('text', 'code', 'reasoning') BEGIN + INSERT INTO messages_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_ad_tool AFTER DELETE ON search_items +WHEN old.content_type = 'tool' BEGIN + INSERT INTO tool_fts(tool_fts, rowid, text) VALUES('delete', old.id, old.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_ad_msg AFTER DELETE ON search_items +WHEN old.content_type IN ('text', 'code', 'reasoning') BEGIN + INSERT INTO messages_fts(messages_fts, rowid, text) VALUES('delete', old.id, old.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_au_tool AFTER UPDATE ON search_items +WHEN old.content_type = 'tool' BEGIN + INSERT INTO tool_fts(tool_fts, rowid, text) VALUES('delete', old.id, old.text); + INSERT INTO tool_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TRIGGER IF NOT EXISTS search_items_au_msg AFTER UPDATE ON search_items +WHEN old.content_type IN ('text', 'code', 'reasoning') BEGIN + INSERT INTO messages_fts(messages_fts, rowid, text) VALUES('delete', old.id, old.text); + INSERT INTO messages_fts(rowid, text) VALUES (new.id, new.text); +END; + +CREATE TABLE IF NOT EXISTS chunks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source_id TEXT NOT NULL, + chunk_idx INTEGER NOT NULL, + content TEXT NOT NULL, + token_count INTEGER NOT NULL, + created_at INTEGER NOT NULL, + embedding BLOB, + UNIQUE(source_id, chunk_idx) +); + +CREATE INDEX IF NOT EXISTS idx_chunks_source_id ON chunks (source_id); + +CREATE TABLE IF NOT EXISTS embedding_metadata ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + chunk_id INTEGER NOT NULL REFERENCES chunks(id) ON DELETE CASCADE, + model_name TEXT NOT NULL, + model_version TEXT NOT NULL, + dimensions INTEGER NOT NULL, + created_at INTEGER NOT NULL +); + +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (1, 'V1 core schema', '1970-01-01 00:00:00', '4e07949ccd3912fb3c0e149be9a2e05fdd51f8cedb8df1f28b3bb5ac5afe532a'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (2, 'V2 embedding tables', '1970-01-01 00:00:00', '37dc9627f01f0e2d0fbea6bba5cd9f609d5da05089eeb9541a057dd2290cf8af'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (3, 'V3 embedding blob column', '1970-01-01 00:00:00', '36cd183f10ff84ab4753be027078cdd710b46efdc830053d4a641af725006ea5'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (4, 'V4 tool_fts trigram index', '1970-01-01 00:00:00', '77e59cf515f33c466282e0f3e921377f584794d0b7cade1704f566374a569a55'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (5, 'V5 drop phantom session_events', '1970-01-01 00:00:00', 'aedaab81efb6bc34d3f664468b71c1a716f5b55d5132156cb43bd7462d549c7b'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (6, 'V6 drop phantom source_metadata column', '1970-01-01 00:00:00', 'a327b9b6e7b8f5fe369c9fc08093ac80a87640daa515890af94b269d430e9378'); +INSERT INTO schema_migrations (version, name, applied_on, checksum) VALUES (7, 'V7 reasoning content_type routes to messages_fts', '1970-01-01 00:00:00', 'a80704442c2a0084f98e4bc53978364b14d1c6bd3bd99ba6119a2a9ecbd685e7'); + + +-- Recovery adapter payload sentinels. +INSERT INTO indexed_files (path, hash, last_indexed) VALUES ('/fixtures/recovery/stranded-v7.jsonl', 'hash-stranded-v7', '2026-08-18T00:00:00Z'); +INSERT INTO indexed_files (path, hash, last_indexed) VALUES ('/fixtures/recovery/stranded-v7-defaults.jsonl', 'hash-stranded-v7-defaults', '2026-08-18T00:00:01Z'); +INSERT INTO search_items (source, source_path, ordinal, role, text, timestamp, uuid, project, content_type) +VALUES ('source-stranded-v7', '/fixtures/recovery/stranded-v7.jsonl', 41, 'assistant', 'text sentinel for stranded-v7', '2026-08-18T12:34:56Z', 'uuid-stranded-v7', 'project-stranded-v7', 'content/stranded-v7'); +INSERT INTO search_items (source, source_path, ordinal, role, text) +VALUES ('session', '/fixtures/recovery/stranded-v7-defaults.jsonl', 42, 'user', 'default sentinel for stranded-v7'); +COMMIT;