diff --git a/CLAUDE.md b/CLAUDE.md index b04c1a1..af7e3f7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -82,12 +82,16 @@ The `SearchEngine` interface is the port; `internal/storage` is the adapter. Dat ### Core Pipeline ``` -validated invocation → canonical startup lock - ├─ owner: prepare/migrate → incremental sync → handler - ├─ read-safe follower: OpenReadOnly snapshot → stderr warning → handler - └─ mutation follower: wait <=5s → owner or retryable sync_in_progress - -Active manifests → mandatory startup sync ───────────────────┐ +validated invocation → startup command class → canonical startup lock + ├─ snapshot-read owner: prepare/migrate → incremental sync → release lease → handler + ├─ snapshot-read follower: OpenReadOnly snapshot → stderr warning → handler + ├─ metadata-read owner: prepare/migrate → incremental sync → release lease → handler + ├─ metadata-read follower: stderr warning → handler without opening the DB + ├─ mutation owner: prepare/migrate → incremental sync → retain lease → handler + ├─ mutation follower: wait <=5s → owner or retryable sync_in_progress + └─ remediation owner: retain mutation-grade lease → skip ordinary compatible-open and pre-handler sync → handler + +Ordinary startup sync inputs ────────────────────────────────┐ JSONL files → fs.WalkDir → SHA-256 dedup → ParseSessions() ─┤ Markdown plans → DiscoverPlanFiles() → ParsePlan() ──────────┤ Declarative Markdown inputs → markdown readers ──────────────┤ @@ -95,6 +99,7 @@ Declarative Markdown inputs → markdown readers ────────── SyncFiles() → perennial SQLite FTS5 │ search/list/patterns/status/validate → database-backed query/output +recover apply → post-install sync under retained remediation lease ``` ### Declarative Markdown Source Types @@ -106,7 +111,7 @@ External knowledge sources are configured with active `*.inputs.toml` manifests - **Defensive parsing**: `SessionRecord` wrapper with `json.RawMessage` for fields handles legacy schemas and noise. - **Noise filtering**: Excludes `system-reminder`, `task-notification`, and subagent sessions by default. - **External FTS5**: Uses `search_items` as content table with SQLite triggers, `snippet()` extraction, and Porter stemmer tokenizer for morphological matching. -- **Mandatory root startup sync**: Every operational command validates active manifests and attempts one incremental sync before executing. Session, plan, and Markdown files are ingestion inputs; SQLite is the perennial record used by search, list, patterns, status, and validate. Use `--source-path` on search as a filter, paired with query text, for database-backed retrieval scoped to a known input path. Concurrent followers are allowed only under the coordinated owner/follower lock contract: a busy read-safe follower uses `OpenReadOnly` on the last committed WAL snapshot, a busy config follower prints validated configuration without opening the DB, and a busy mutation follower waits up to five seconds before failing retryably with `sync_in_progress`. +- **Startup command classes**: Operational commands are explicitly classified as `snapshot-read: search, list, patterns, status, validate`; `metadata-read: config`; `mutation: annotate, purge, rebuild`; and `remediation: recover`. Snapshot-read, metadata-read, and mutation owners validate active manifests, prepare/migrate the index, and attempt one incremental sync before their handlers. Metadata-read followers can print validated configuration without opening the DB. Remediation (`recover`) acquires and retains the mutation-grade startup lock, but deliberately skips ordinary compatible-open/index preparation and pre-handler sync so recovery can inspect or replace an index that normal startup may reject; non-dry-run apply runs post-install sync under the same retained lease before printing the recovery report. Session, plan, and Markdown files are ingestion inputs; SQLite is the perennial record used by search, list, patterns, status, and validate. Use `--source-path` on search as a filter, paired with query text, for database-backed retrieval scoped to a known input path. Concurrent followers are allowed only under the coordinated owner/follower lock contract: a busy read-safe follower uses `OpenReadOnly` on the last committed WAL snapshot, a busy config follower prints validated configuration without opening the DB, and busy mutation/remediation followers wait up to five seconds before failing retryably with `sync_in_progress`. - **Incremental sync**: SHA-256 hash per file stored in `indexed_files` table; unchanged files are skipped. - **Plan indexing**: Markdown plans from `~/.claude/plans/` split by `##` headers, each section indexed as a separate search item with `source='plan'`. - **Declarative Markdown inputs**: `markdown_document` and `markdown_sections` readers reuse `internal/sources` parsers, flow through the normal input manifest registry/autosync path, and store the manifest `source` value on indexed rows. They do not parse YAML frontmatter into structured metadata. @@ -118,7 +123,7 @@ External knowledge sources are configured with active `*.inputs.toml` manifests - **Content-type classification**: Messages classified as `text`/`code`/`tool`/`reasoning` based on message content types during sync. Tool content is indexed in separate `search_items` rows with `content_type='tool'`. Pi agent reasoning blocks are captured when `index_reasoning=true` (default off) in the input manifest and indexed with `content_type='reasoning'`. Sync writes only to `search_items`; the `session_events` table was dropped in migration v5. - **Split FTS by retrieval semantics**: tool content (`content_type='tool'`) lives in a separate FTS5 index `tool_fts` (tokenizer `trigram`, substring/exact match for paths/commands/errors); prose content (text, code, reasoning) lives in `messages_fts` (`porter unicode61`). Migration v4 branched the triggers by content type. Migration v7 updated the triggers to route 'reasoning' alongside 'text'/'code' to `messages_fts`. `--content-type tool` queries `tool_fts`; prose queries `messages_fts`; an unfiltered query merges both via Reciprocal Rank Fusion (RRF, k=60), which fuses by rank position, not score magnitude, and is immune to incomparable cross-tokenizer BM25 scales. The trigram tokenizer matches substrings of ≥3 characters, so tool queries shorter than 3 characters will match zero results. - **Pure Go SQLite**: `modernc.org/sqlite` — no CGO, trivially cross-compilable. -- **Whitespace-insensitive schema normalization (fix #52 part 1, PR #56)**: `normalizeSQL()` handles SQL lexical structure: line comments (`--...`), block comments (`/* ... */` non-nesting), single-quoted literals (with `''` escapes), double-quoted identifiers (with `""` escapes), backtick and bracket identifiers. Defect #1 (PR #56): apostrophes inside comments no longer flip quote state. Defect #2 (PR #56): double-quoted identifiers preserve internal whitespace (`"my table"` ≠ `"my table"`). Defect #3 (PR #56): comment-internal whitespace is collapsed (e.g., `-- note double` → `-- note double`), preventing comment formatting from being load-bearing. CRLF line endings in comments are normalized to LF for deterministic signatures across line ending styles. Block comments always have a single space before `*/` to preserve structure. This eliminates cosmetic DDL formatting (e.g., `ALTER TABLE ADD COLUMN` inline vs. hand-wrapped multi-line, or comments with varying spacing) as a source of signature mismatch. All 76 manifest signatures are regenerated via `RegenerateManifestJSON` under the new normalization (reproducible: load fixtures, compute signatures under current `normalizeSQL`, write back manifest.json). The fix prevents false `unsupported_lineage` rejections when published-release-migrated databases encounter the catalog. +- **Semantic schema lineage invariant (fixes #52/#58)**: Schema identity is the pair `(AppliedVersion, semantic Signature)`. Migration rows are provenance only and are never signature input. Canonical SQL is deliberately conservative: it discards comments and formatting, but preserves literals, quoted identifiers, constraints, expressions, triggers, indexes, and virtual-table configuration. Cosmetic DDL layout changes must not create a new identity, while any semantic schema change must. Every physical catalog fixture is an upgrade target; no recognized fixture may be skipped. Every new migration must pass `TestEveryCatalogFixtureReachesCurrentSemanticHead` before merge, proving all known physical lineages can reach the current semantic head. - **Startup coordination**: `github.com/gofrs/flock` via `internal/startuplock` — persistent `.startup-sync.lock` sidecar with `0600` permissions, OS-owned advisory lock, local-host-only WAL snapshot coordination, and read-only followers that never delete the sidecar. - **Connection pragmas via `_pragma`**: `modernc.org/sqlite` honors DSN pragmas only in the `_pragma=name(value)` form; the mattn-style `_name=value` (e.g. `_busy_timeout=5000`, `_journal_mode=WAL`) is silently ignored, which had left the DB in rollback (delete) journal mode with a zero busy timeout — the root cause of `database is locked` (SQLITE_BUSY) errors. Both connections in `internal/storage/storage.go` set `_pragma=journal_mode(WAL)`, `_pragma=synchronous(NORMAL)`, and `_pragma=busy_timeout(5000)` (read-only sets only the busy timeout; journal mode is persisted in the file). Always use `_pragma=name(value)` for any new connection pragma here. - **Autoupdate**: `picokit/autoupdate` fetches and stages the latest GitHub release in the background; `run()` waits up to 10s after the command completes so short-lived commands don't kill the download before it finishes. Autoupdate is mandatory: there is no runtime opt-out. `newUpdater()` in `cmd/backscroll/main.go` is the single wiring point — it calls `autoupdate.New` with no `envDisable`, and both `run()` and the wiring test go through it, so re-adding an opt-out would fail the test. Dev builds are exempt by identity — a plain `go build` yields `version="dev"`, which picokit never fetches or applies — so validate against a dev build, not an env var. `scripts/eval.sh` builds its own dev binary for the same reason (a release binary would fetch+wait ~10s per invocation). @@ -126,7 +131,7 @@ External knowledge sources are configured with active `*.inputs.toml` manifests - **Schema migration rule**: Every new table or column MUST be introduced as a new migration version (increment the version number and add a new version-check block in `SetupSchema()`). Never modify existing migration blocks — existing databases that already passed that version will never re-run them. Migration v5 drops the phantom `session_events` table (and its indexes `idx_session_events_order` and `idx_session_events_project`) — the table was write-only dead weight after structured-stats filtering was removed. Migration v6 drops the phantom `search_items.source_metadata` column via `ALTER TABLE ... DROP COLUMN` — it had a setter but zero production callers and was never read. Migration v13 adds indexes on `template_matches(source_path)` and `correction_signals(source_path)` to optimize backfill discovery queries (reduces O(N·M) subquery scans to O(N·log M) index lookups). Migration count remains v1–v13; this coordination update does not add a migration. - **F0a rich capture (migration v8)**: readers extract per-message identity and tool metadata BEFORE serialization/cleaning destroys the evidence — `uuid` (record uuid; tool blocks get stable `#tN`/`#rN` suffixes by block index), `tool_name`, `command_head`, `is_error` (`*bool`, three-valued: tool_result blocks carry it and it is paired back onto the tool_use message cross-record via `tool_use_id`), and `was_interrupted` (detected on raw content before `CleanContent` strips "Request interrupted"). Persisted to `search_items` (`extraction_version`, `was_interrupted` columns) and the perennial `tool_events` satellite table (`UNIQUE(source_path, ordinal)`, no CASCADE lifecycle — only `purge` deletes from it, explicitly). Claude reader only; Pi/OpenCode emit zero values and stay on the legacy path. Design: `docs/superpowers/specs/2026-07-17-pattern-discovery-northstar-design.md`. - **F0a.1 command-head extraction v2 (extraction_version bump to 2)**: The `commandHead()` function in `internal/readers/claude_reader.go` now strips leading POSIX variable assignments (tokens containing `=`) before extracting the actual command name. Example: `"SP=/path/to/code; go test ./..."` extracts `"go"` instead of `"SP=/path/to/code;"`. This reduces noise in sequence mining and command pattern discovery. The bump to `extraction_version=2` triggers B1 backfill on next sync, incrementally re-extracting command_head from stored text for files indexed before this change (up to 200 files/run). -- **F0b perennial sync**: the DB is the perennial event store — session JSONL files expire (~30 days), indexed sessions survive them. Session files where EVERY message has a uuid sync append-only (no DELETE; `INSERT OR IGNORE` + UNIQUE constraints; row ids stable forever), with a one-time transition cleanup of legacy uuid-NULL rows per file. Files with any uuid-less message (Pi/OpenCode, legacy Claude) keep wipe-and-reload. `rebuild` is NON-destructive: after the mandatory root startup sync has prepared the database, the handler re-derives both FTS indexes from `search_items` via FTS5 external-content `'rebuild'` in one transaction (`RebuildFTS()`) and performs no second sync — it never deletes rows and never re-reads disk as source of truth. `purge --before` is the only deletion path and deletes `tool_events` satellites explicitly in the same transaction (no CASCADE). The recover flow retains the startup lock through post-install sync. +- **F0b perennial sync**: the DB is the perennial event store — session JSONL files expire (~30 days), indexed sessions survive them. Session files where EVERY message has a uuid sync append-only (no DELETE; `INSERT OR IGNORE` + UNIQUE constraints; row ids stable forever), with a one-time transition cleanup of legacy uuid-NULL rows per file. Files with any uuid-less message (Pi/OpenCode, legacy Claude) keep wipe-and-reload. `rebuild` is NON-destructive: after mutation-class startup sync has prepared the database, the handler re-derives both FTS indexes from `search_items` via FTS5 external-content `'rebuild'` in one transaction (`RebuildFTS()`) and performs no second sync — it never deletes rows and never re-reads disk as source of truth. `purge --before` is the only deletion path and deletes `tool_events` satellites explicitly in the same transaction (no CASCADE). The recover flow retains the remediation startup lock through post-install sync. - **F1 exit code mining**: the **reader** parses the exit code from the FULL tool output before `toolfmt` truncates it (`SerializeToolOutput` caps at 4000 runes and a Bash exit code is usually on the last line, so parsing the capped text lost exactly the codes worth having). `ExtractExitCodeText` does the parsing with no tool gate, because a `tool_result` block lives in a different JSONL record than its `tool_use` and the tool name is not knowable at that point; the Bash gate and the code are applied when result and use are paired by `tool_use_id` — the same cross-record pairing already used for `is_error`. `SyncFiles` persists `IndexedMessage.ExitCode` verbatim and MUST NOT re-derive a code from `msg.Text`, which is truncated. The extracted exit code is stored in the `tool_events.exit_code` column (migration v8; NULL for non-Bash tools or no match). The `patterns` command aggregates tool_events by (tool_name, command_head) for commands or (tool_name, is_error, exit_code) for failures, returning top N sorted by frequency with optional filters by project, session tag, and time window. Coverage metric reports the count of events with non-NULL is_error (signalled events) against the total failure count in the result set. - **F2 template mining (migration v10)**: unsupervised Drain-inspired template miner (`internal/templates/Miner`) discovers recurring error patterns from tool output during sync. Miner uses fixed-depth token prefix clustering (depth=2) to group messages; beyond the prefix, numeric/path/UUID tokens become `<*>` variables. Error-bearing lines (is_error=true) are extracted per tool via `ExtractErrorLines` (calibrated per tool: Bash prefers LAST non-empty line + error-matching; Go test matches "--- FAIL:", "FAIL\t", "error:"; others default to error-matching heuristic) and deterministically mined with SHA256 signature. Templates stored in `message_templates` (signature, normalization_version, template_text, occurrence_count, first_seen, last_seen) joined via `template_matches` (template_id, source_path, ordinal, item_uuid) with UNIQUE constraint for idempotency. Mining runs inside `SyncFiles` transaction; re-syncing increments occurrence_count only for new matches (detected via INSERT OR IGNORE). Query method `AggregateTemplates(opts)` filters by min_support (default 3), project, date range; `patterns --kind templates [--min-support N]` exposes results in text/JSON/robot formats with normalization_version metadata. _Q1 backfill update:_ Backfill mining filters rows by: include a tool-text row only when (it has case-insensitive "error: " prefix OR its ordinal has a tool_events row with is_error=1) AND it is not an input serialization. - **One line-selection predicate for both mining paths**: `shouldMineToolLine(contentType, text, isError)` in `internal/storage/mining.go` is called by sync-time `mineTemplatesForFile` and by backfill. Only the error signal differs — sync reads `IsError` off the message, backfill derives it from an `"error: "` prefix or a `tool_events` row. The predicate exists because the two paths had drifted, and the drift was the bug: sync selected on `ToolName != ""`, which picks the **tool_use** message, whose text *is* the input serialization, while the error text lives on the **tool_result** message that carries no `ToolName`. Sync therefore mined inputs and never errors. Tool results have no tool name, so sync mines them as `"Unknown"`, matching backfill; per-tool line calibration is a follow-up. @@ -143,14 +148,14 @@ External knowledge sources are configured with active `*.inputs.toml` manifests - **B1 extraction-version backfill (incremental re-sync)**: Files indexed before v8 carry no rich metadata (uuid NULL, no tool_events, no corrections). A new storage query `StalePaths(currentVersion)` returns paths whose rows have `extraction_version IS NULL` or `< currentVersion`. In `maybeAutoSync`, a stale-set is built once per run and during hash evaluation, unchanged-hash files in the stale-set re-parse anyway (skip the `continue`), up to a per-run cap (default 200 files). The existing perennial path + transition cleanup (sync.go:99-108) then does the right thing automatically — legacy uuid-NULL rows are deleted (one-time), rich uuid-bearing rows replace them via `INSERT OR IGNORE`, and `extraction_version` updates. Repeated `maybeAutoSync` invocations drain the backlog incrementally FIFO (ordered by `last_indexed` ASC). Re-parsing stops when expired JSONL files vanish from disk — the database survives them (perennity contract). - **B2 project fallback identity + historical re-resolution**: when the global registry (`~/.config/backscroll/projects.toml`) does not match a session's cwd, `projects.Identify()` derives a sanitized fallback id from the cwd basename (lowercase, `[a-z0-9-_]`; registry always wins when it matches). `rebuild` runs a re-resolution pass: `ReresolveProjects` decodes Claude's session-dir encoding (dashes-for-slashes; if the decoded path exists on disk its basename wins, else the last dash segment — the encoding is lossy and this heuristic is documented as ambiguous for dir names containing dashes) and relabels rows stuck at `project='unknown'`, returning distinct files resolved. Fallback labels are not revisited once set — followed by registry-aware re-resolution below. - **B2.1 registry-aware re-resolution**: `projects.Identify()` now carries a `FromRegistry` flag to distinguish registry matches from fallback labels. `rebuild` includes a second phase that loads `~/.config/backscroll/projects.toml` and re-resolves historical sessions labeled with fallback IDs. For each session path, it decodes the cwd, applies cross-host normalization, and calls `Identify` with the registry. If a registry entry matches (FromRegistry=true) and differs from the stored fallback, the rows are updated. Only registry matches count — fallback-only paths are not re-labeled (no churn). This allows a future registry entry to correct historical misattribution without touching already-correct sessions. Wired into rebuild via `ReresolveProjectsWithRegistry` (queries.go). -- **B3 retroactive mining over stored text**: for sessions that expire from disk before Template/Correction/Sequence mining runs, `BackfillDerived()` recovers templates, correction signals, and lossy tool_events from stored text in search_items. Stale template re-mining (F2a) is integrated: BackfillDerived first discovers stale paths via `StaleTemplatePaths(CurrentNormalizationVersion)` and re-mines templates under v2 heuristics, updating existing templates to normalization_version=2. Template mining reuses `internal/templates/Miner` over tool-text rows. Correction detection filters input to prose only (role='user' AND content_type IN ('text','code')) to avoid tool_result false positives; lexicon, denial, and rephrase detectors run on prose; interrupt detector runs on all user messages. Lossy tool_events reverse-parse toolfmt input serialization ( ...; heuristic: first token has no '=', at least one subsequent has '=') to extract tool_name and command_head; outputs unattributable without tool_use_id so skipped; extraction_version=0 marks lossy rows (inputs only, unrecoverable once disk source expires). `rebuild` command now runs after mandatory root startup sync and then: (1) re-derives FTS from DB, (2) BackfillDerived (stale-path discovery + re-mining + expired-file discovery + batch mining + progress reporting), (3) re-resolves projects, and (4) applies registry-aware re-resolution. It performs no second sync. The recover path retains the lock through post-install sync. -- **Early input validation**: CLI commands validate positional arguments and semantic flag values through Cobra `Args`, which runs before the root `PersistentPreRunE`; the root also checks required flags and flag groups before invoking the mandatory startup policy. Invalid invocations therefore fail before opening, creating, or synchronizing the database. Keep pure validators reusable from direct `run*` callers so tests and non-Cobra entry points preserve the same boundary. +- **B3 retroactive mining over stored text**: for sessions that expire from disk before Template/Correction/Sequence mining runs, `BackfillDerived()` recovers templates, correction signals, and lossy tool_events from stored text in search_items. Stale template re-mining (F2a) is integrated: BackfillDerived first discovers stale paths via `StaleTemplatePaths(CurrentNormalizationVersion)` and re-mines templates under v2 heuristics, updating existing templates to normalization_version=2. Template mining reuses `internal/templates/Miner` over tool-text rows. Correction detection filters input to prose only (role='user' AND content_type IN ('text','code')) to avoid tool_result false positives; lexicon, denial, and rephrase detectors run on prose; interrupt detector runs on all user messages. Lossy tool_events reverse-parse toolfmt input serialization ( ...; heuristic: first token has no '=', at least one subsequent has '=') to extract tool_name and command_head; outputs unattributable without tool_use_id so skipped; extraction_version=0 marks lossy rows (inputs only, unrecoverable once disk source expires). `rebuild` command now runs after mutation-class startup sync and then: (1) re-derives FTS from DB, (2) BackfillDerived (stale-path discovery + re-mining + expired-file discovery + batch mining + progress reporting), (3) re-resolves projects, and (4) applies registry-aware re-resolution. It performs no second sync. The recover path retains the remediation lock through post-install sync. +- **Early input validation**: CLI commands validate positional arguments and semantic flag values through Cobra `Args`, which runs before the root `PersistentPreRunE`; the root also checks required flags and flag groups before invoking the startup policy for the command class. Invalid invocations therefore fail before opening, creating, synchronizing, or recovering the database. Keep pure validators reusable from direct `run*` callers so tests and non-Cobra entry points preserve the same boundary. - **Coverage gate**: CI enforces ≥85% aggregate statement coverage via `go test ./... -race -coverprofile`. Local check: `bash scripts/check-coverage.sh`. Tests that depend on local machine state (e.g. `~/.config/backscroll/projects.toml`) must use `t.Setenv("HOME", tempDir)` to stay reproducible on CI. Likewise, `InputsDir` branches requiring `BACKSCROLL_CONFIG_DIR` to be unset must set it to `""` via `t.Setenv`. To test the `Validate` orphan path, insert directly into `search_items` without a matching `indexed_files` row. - **Zero-result guidance**: when `search`/`list` return no rows, actionable suggestions (`--all-projects`, `--content-type tool`, `backscroll status`) are printed to STDERR — never STDOUT, so `--json` stays a clean empty payload. - **Search robot output contract**: robot mode on search emits `result_N_field=value` lines exactly once-wrapped (the robot path writes lines directly; passing pre-formatted lines through the picokit formatter double-wraps them as `result_N=result_N_field=...`). Search robot string values escape backslash as `\\`, carriage return as `\r`, and newline as `\n`. - **Cross-host project identity**: `projects.Identify()` normalizes session cwd against registry roots by matching root tails (≥2 components, case-insensitive), so `/home/shared/` sessions resolve against `/Users/Shared/` roots on a synced index. Registry roots should keep distinct suffixes — two projects whose roots share the same trailing components could misbucket. - **Recall eval-set**: `docs/eval/queries.toml` (~20 real mined queries with `expected_match` ground truth) + `scripts/eval.sh` compute recall@5; a query counts only if its expected target appears in the top 5. Local regression gate, not a required CI step. -- **Single catalog source of truth (fix #52)**: The lineage catalog lives in `internal/compat/manifest.json` (data, regenerable via `REGEN_MANIFEST=1`). `internal/storage/recovery_records.go` queries `compat.Catalog.IsKnownSignature()` to check if a schema is recognized, rather than carrying a stale hardcoded switch. This eliminates duplication and the risk of signature drift after normalization changes. Whitespace normalization collapsed 76 fixtures into 17 distinct signatures; collision consistency is verified by a test that ensures all entries with the same signature agree on `AppliedVersion` and `HasSourceMetadata`. +- **Single catalog source of truth**: The lineage catalog lives in `internal/compat/manifest.json` (data, regenerable via `REGEN_MANIFEST=1`). `internal/storage/recovery_records.go` queries `compat.Catalog.IsKnownSignature()` to check if a schema is recognized, rather than carrying a stale hardcoded switch. This eliminates duplication and the risk of catalog drift. Catalog validation must confirm semantic-signature collisions remain compatible for all entries sharing a signature. - **V14 file metadata prefilter with racy-clean guard**: (migration v14) adds `file_size INTEGER` and `file_mtime TEXT` columns to `indexed_files`, populated during sync for every processed file. During startup sync, `maybeAutoSync` collects current file metadata (size via `os.Stat()`, mtime in RFC3339 format) BEFORE calling `reader.Hash()` on each discovered file. Freshness guarantee: skipped files have not changed since last index time; any modification (content, size, timestamp) re-triggers hashing. Conservative design: NULL metadata for pre-v14 rows means always re-hash. Racy-clean guard (git-style): files whose mtime is not strictly older than `last_indexed` (within 2-second granularity margin) are treated as potentially modified in the same timestamp tick and are always re-hashed, preventing silent data loss from same-length edits. Unchanged files (size + mtime both match, and mtime strictly older than `last_indexed`) skip re-reading entirely, reducing startup time on stable corpora. Truncations, replacements, and same-size-different-content edits are all reliably detected. The `isRacyCleanFile()` function accepts both SQLite CURRENT_TIMESTAMP raw format (`2006-01-02 15:04:05`) and RFC3339 (actual driver conversion): the indexed_files.last_indexed column stores raw text but modernc.org/sqlite converts DATETIME on read, so Go receives RFC3339; the sqlite3 CLI shows raw format, a trap that has fooled multiple reviewers, so verify through the driver. ## Dependencies diff --git a/cmd/backscroll/compat_diagnostics_test.go b/cmd/backscroll/compat_diagnostics_test.go index 44dfbdf..13664ee 100644 --- a/cmd/backscroll/compat_diagnostics_test.go +++ b/cmd/backscroll/compat_diagnostics_test.go @@ -7,14 +7,12 @@ import ( "encoding/json" "errors" "fmt" - "io" "os" "path/filepath" "strings" "testing" "github.com/pablontiv/backscroll/internal/compat" - "github.com/pablontiv/backscroll/internal/config" "github.com/pablontiv/backscroll/internal/storage" ) @@ -540,7 +538,7 @@ func TestLiveWALStartupUsesCompatibleIndexWithoutRecoveryDiagnostic(t *testing.T } } -func TestRecoveryContinuationExecutesInConfiguredSamePathContextWithEmptyWAL(t *testing.T) { +func TestRecoverDryRunBypassesPreparationForAlterBuiltLineage(t *testing.T) { dbPath := newFixtureIndexDB(t, "v13-development-alter-built.sql") setIndexPolicyEnv(t, dbPath, t.TempDir()) walPath := dbPath + "-wal" @@ -550,40 +548,25 @@ func TestRecoveryContinuationExecutesInConfiguredSamePathContextWithEmptyWAL(t * before := snapshotSQLiteFiles(t, dbPath) walBefore, err := os.Stat(walPath) if err != nil { - t.Fatalf("stat empty WAL before continuation: %v", err) + t.Fatalf("stat empty WAL before dry-run: %v", err) } - emptyInputs := filepath.Join(t.TempDir(), "empty-inputs") - if err := os.MkdirAll(emptyInputs, 0o755); err != nil { - t.Fatalf("mkdir empty recovery inputs: %v", err) - } - cfg := &config.Config{DatabasePath: dbPath, SessionDirs: []string{emptyInputs}} - startupDiagnostic := continuationFor(compat.Diagnostic{Code: compat.CodeUnsupportedLineage, Summary: "fixture diagnostic"}, dbPath) - startupErr := indexDiagnosticError{diagnostic: startupDiagnostic} - - var stdout, stderr bytes.Buffer - root := buildRootCmdWithStartup(&stdout, &stderr, func(context.Context, io.Writer, startupCommandClass) startupResult { - return startupResult{Config: cfg, Failure: &startupFailure{ - Stage: startupStageIndexPrepare, - Cause: startupErr, - Diagnostic: startupDiagnostic, - Recoverable: true, - }} - }) - root.SetArgs(startupDiagnostic.Continuation) - if err := root.Execute(); err != nil { - t.Fatalf("execute continuation %v after startup diagnostic %s: %v\nstdout=%q stderr=%q", startupDiagnostic.Continuation, startupDiagnostic.Code, err, stdout.String(), stderr.String()) + stdout, stderr, err := runCmd("recover", "--from", dbPath, "--dry-run") + if err != nil { + t.Fatalf("recover dry-run failed: %v\nstdout=%q stderr=%q", err, stdout, stderr) } - if !strings.Contains(stdout.String(), "recovery dry run") { - t.Fatalf("continuation output = %q, want recovery dry run", stdout.String()) + if !strings.Contains(stdout, "recovery dry run") { + t.Fatalf("stdout=%q", stdout) } - if stderr.Len() != 0 { - t.Fatalf("continuation stderr = %q, want empty", stderr.String()) + for _, forbidden := range []string{"migration_failed", "unsupported_lineage", "f6a081b9", "50016 diagnostic"} { + if strings.Contains(stdout+stderr, forbidden) { + t.Fatalf("output retained %q: stdout=%q stderr=%q", forbidden, stdout, stderr) + } } assertSQLiteFilesUnchanged(t, dbPath, before) walAfter, err := os.Stat(walPath) if err != nil { - t.Fatalf("stat empty WAL after continuation: %v", err) + t.Fatalf("stat WAL after dry-run: %v", err) } if walAfter.Size() != 0 || walAfter.Mode() != walBefore.Mode() || !walAfter.ModTime().Equal(walBefore.ModTime()) { t.Fatalf("empty WAL metadata changed: before=%+v after=%+v", walBefore, walAfter) diff --git a/cmd/backscroll/index_policy_test.go b/cmd/backscroll/index_policy_test.go index 45b5705..3401a20 100644 --- a/cmd/backscroll/index_policy_test.go +++ b/cmd/backscroll/index_policy_test.go @@ -143,26 +143,25 @@ func TestPrepareIndexDataReadReturnsReadOnlyConnection(t *testing.T) { } func TestPrepareIndexDataReadDoesNotApplyPendingMigration(t *testing.T) { - dbPath := filepath.Join(t.TempDir(), "index.db") - writer, err := storage.Open(dbPath) - if err != nil { - t.Fatal(err) - } - if _, err := writer.DB().Exec(`DELETE FROM schema_migrations WHERE version = 13`); err != nil { - t.Fatal(err) - } - if err := writer.Close(); err != nil { - t.Fatal(err) - } + dbPath := newFixtureIndexDB(t, "v13.sql") db, diag, err := prepareIndex(context.Background(), &config.Config{DatabasePath: dbPath}, indexDataRead) if db != nil { _ = db.Close() t.Fatal("read preparation returned DB requiring migration") } - if err == nil && diag == nil { + if err != nil { + t.Fatalf("read preparation returned unexpected error: %v", err) + } + if diag == nil { t.Fatal("read preparation accepted pending migration") } + if diag.Code != compat.CodeIndexStale { + t.Fatalf("read preparation diagnostic code=%q, want %q", diag.Code, compat.CodeIndexStale) + } + if !strings.Contains(diag.Summary, "migration") || !strings.Contains(diag.Summary, "step") { + t.Fatalf("read preparation diagnostic summary=%q, want pending migration-step summary", diag.Summary) + } inspect, err := storage.OpenReadOnly(dbPath) if err != nil { @@ -170,11 +169,37 @@ func TestPrepareIndexDataReadDoesNotApplyPendingMigration(t *testing.T) { } defer inspect.Close() var count int - if err := inspect.DB().QueryRow(`SELECT COUNT(*) FROM schema_migrations WHERE version = 13`).Scan(&count); err != nil { + if err := inspect.DB().QueryRow(`SELECT COUNT(*) FROM schema_migrations WHERE version = 14`).Scan(&count); err != nil { t.Fatal(err) } if count != 0 { - t.Fatalf("read path applied migration 13 count=%d", count) + t.Fatalf("read path applied migration 14 count=%d", count) + } + assertIndexedFilesColumnAbsent(t, inspect.DB(), "file_size") + assertIndexedFilesColumnAbsent(t, inspect.DB(), "file_mtime") +} + +func assertIndexedFilesColumnAbsent(t *testing.T, db *sql.DB, column string) { + t.Helper() + rows, err := db.Query(`PRAGMA table_info(indexed_files)`) + if err != nil { + t.Fatalf("indexed_files table_info: %v", err) + } + defer rows.Close() + for rows.Next() { + var cid int + var name, typ string + var notNull, pk int + var defaultValue sql.NullString + if err := rows.Scan(&cid, &name, &typ, ¬Null, &defaultValue, &pk); err != nil { + t.Fatalf("scan indexed_files column: %v", err) + } + if name == column { + t.Fatalf("read path applied migration 14 column %s", column) + } + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate indexed_files columns: %v", err) } } diff --git a/cmd/backscroll/recover.go b/cmd/backscroll/recover.go index 70950ec..87e2b94 100644 --- a/cmd/backscroll/recover.go +++ b/cmd/backscroll/recover.go @@ -1,7 +1,6 @@ package main import ( - "errors" "fmt" "io" "strings" @@ -33,12 +32,11 @@ func newRecoverCmd(stdout, stderr io.Writer) *cobra.Command { }, RunE: func(cmd *cobra.Command, args []string) error { startup := startupResultFrom(cmd) - startupFailure := optionalStartupFailureError(startup.startupFailure()) cfg := startup.Config if cfg == nil { loaded, err := config.Load() if err != nil { - return errors.Join(startupFailure, fmt.Errorf("load config for recovery: %w", err)) + return fmt.Errorf("load config for recovery: %w", err) } cfg = loaded } @@ -51,7 +49,7 @@ func newRecoverCmd(stdout, stderr io.Writer) *cobra.Command { if backupPath, ok := recovery.RestorableBackupPath(err); ok { _, _ = fmt.Fprintf(stderr, "manual recovery backup path: %s\n", backupPath) } - return errors.Join(startupFailure, fmt.Errorf("recovery failed: %w", err)) + return fmt.Errorf("recovery failed: %w", err) } if !dryRun { if err := recoverPostInstallSync(cfg, stderr); err != nil { @@ -63,7 +61,7 @@ func newRecoverCmd(stdout, stderr io.Writer) *cobra.Command { if report.BackupPath != "" { _, _ = fmt.Fprintf(stderr, "manual recovery backup path: %s\n", report.BackupPath) } - return errors.Join(startupFailure, fmt.Errorf("post-recovery sync: %w", err)) + return fmt.Errorf("post-recovery sync: %w", err) } } printRecoveryReport(stdout, report, dryRun) diff --git a/cmd/backscroll/recover_test.go b/cmd/backscroll/recover_test.go index 3502c56..9bb2c0b 100644 --- a/cmd/backscroll/recover_test.go +++ b/cmd/backscroll/recover_test.go @@ -167,8 +167,7 @@ func TestRecoverDryRunSkipsPostInstallSync(t *testing.T) { } } -func TestRecoverPostInstallSyncFailurePreservesStartupCause(t *testing.T) { - startupErr := errors.New("injected startup failure") +func TestRecoverPostInstallSyncFailurePreservesSyncCause(t *testing.T) { syncErr := errors.New("injected post-sync failure") cfg := &config.Config{DatabasePath: filepath.Join(t.TempDir(), "active.db")} installedPath := cfg.DatabasePath + ".installed" @@ -188,21 +187,17 @@ func TestRecoverPostInstallSyncFailurePreservesStartupCause(t *testing.T) { var stdout, stderr bytes.Buffer root := buildRootCmdWithStartup(&stdout, &stderr, func(context.Context, io.Writer, startupCommandClass) startupResult { - return startupResult{Config: cfg, Failure: &startupFailure{ - Stage: startupStageStartupSync, - Cause: startupErr, - Diagnostic: continuationFor(compat.Diagnostic{Code: compat.CodeIndexStale, Summary: startupErr.Error()}, cfg.DatabasePath), - Recoverable: true, - }} + return startupResult{Config: cfg} }) root.SetArgs([]string{"recover", "--from", "stranded.db"}) err := root.Execute() - if !errors.Is(err, startupErr) { - t.Fatalf("error=%v does not preserve startup failure", err) - } if !errors.Is(err, syncErr) { t.Fatalf("error=%v does not preserve post-sync failure", err) } + var failure *startupFailure + if errors.As(err, &failure) { + t.Fatalf("error=%v unexpectedly matches startupFailure target %#v", err, failure) + } if stdout.Len() != 0 { t.Fatalf("report printed before failed post-sync: %q", stdout.String()) } @@ -216,35 +211,6 @@ func TestRecoverPostInstallSyncFailurePreservesStartupCause(t *testing.T) { } } -func TestRecoverSuccessfulContinuationRemediatesStartupFailure(t *testing.T) { - startupErr := errors.New("injected startup failure") - cfg := &config.Config{DatabasePath: filepath.Join(t.TempDir(), "active.db")} - - originalExecute := recoverExecute - recoverExecute = func(context.Context, recovery.Options) (recovery.Report, error) { - return recovery.Report{ActivePath: cfg.DatabasePath}, nil - } - t.Cleanup(func() { recoverExecute = originalExecute }) - - originalPostInstallSync := recoverPostInstallSync - recoverPostInstallSync = func(*config.Config, io.Writer) error { return nil } - t.Cleanup(func() { recoverPostInstallSync = originalPostInstallSync }) - - var stdout, stderr bytes.Buffer - root := buildRootCmdWithStartup(&stdout, &stderr, func(context.Context, io.Writer, startupCommandClass) startupResult { - return startupResult{Config: cfg, Failure: &startupFailure{ - Stage: startupStageStartupSync, - Cause: startupErr, - Diagnostic: continuationFor(compat.Diagnostic{Code: compat.CodeIndexStale, Summary: startupErr.Error()}, cfg.DatabasePath), - Recoverable: true, - }} - }) - root.SetArgs([]string{"recover", "--from", "stranded.db"}) - if err := root.Execute(); err != nil { - t.Fatalf("recover returned startup failure after successful remediation: %v", err) - } -} - func TestRecoverDryRunMatchesUnionApplyPlanWithoutWrites(t *testing.T) { dir := t.TempDir() home := filepath.Join(dir, "home") diff --git a/cmd/backscroll/startup_commands.go b/cmd/backscroll/startup_commands.go index da03a78..34d7097 100644 --- a/cmd/backscroll/startup_commands.go +++ b/cmd/backscroll/startup_commands.go @@ -8,6 +8,7 @@ const ( startupSnapshotRead startupCommandClass = "snapshot-read" startupMetadataRead startupCommandClass = "metadata-read" startupMutation startupCommandClass = "mutation" + startupRemediation startupCommandClass = "remediation" startupClassKey = "backscroll.io/startup-class" ) @@ -17,25 +18,29 @@ func startupCommandClassFor(cmd *cobra.Command) (startupCommandClass, bool) { } class := startupCommandClass(cmd.Annotations[startupClassKey]) switch class { - case startupSnapshotRead, startupMetadataRead, startupMutation: + case startupSnapshotRead, startupMetadataRead, startupMutation, startupRemediation: return class, true default: return startupMutation, false } } +func startupClassRetainsLease(class startupCommandClass) bool { + return class == startupMutation || class == startupRemediation +} + func registerStartupCommand(root *cobra.Command, class startupCommandClass, cmd *cobra.Command) { if cmd.Annotations == nil { cmd.Annotations = make(map[string]string) } cmd.Annotations[startupClassKey] = string(class) - if class == startupMutation { - cmd.RunE = wrapMutationRunE(cmd.RunE) + if startupClassRetainsLease(class) { + cmd.RunE = wrapLeaseRetainingRunE(cmd.RunE) } root.AddCommand(cmd) } -func wrapMutationRunE(runE func(*cobra.Command, []string) error) func(*cobra.Command, []string) (retErr error) { +func wrapLeaseRetainingRunE(runE func(*cobra.Command, []string) error) func(*cobra.Command, []string) (retErr error) { return func(cmd *cobra.Command, args []string) (retErr error) { defer func() { retErr = startupResultFrom(cmd).release(retErr) }() return runE(cmd, args) diff --git a/cmd/backscroll/startup_commands_test.go b/cmd/backscroll/startup_commands_test.go index 10152ae..b7673a3 100644 --- a/cmd/backscroll/startup_commands_test.go +++ b/cmd/backscroll/startup_commands_test.go @@ -21,7 +21,7 @@ func TestEveryOperationalCommandHasApprovedStartupClass(t *testing.T) { "annotate": startupMutation, "purge": startupMutation, "rebuild": startupMutation, - "recover": startupMutation, + "recover": startupRemediation, } if len(root.Commands()) != len(want) { t.Fatalf("operational command count=%d want=%d", len(root.Commands()), len(want)) @@ -38,30 +38,62 @@ func TestEveryOperationalCommandHasApprovedStartupClass(t *testing.T) { } } -func TestMutationRegistrationReleasesStartupLease(t *testing.T) { +func TestLeaseRetainingRegistrationReleasesStartupLease(t *testing.T) { handlerErr := errors.New("handler failed") - for _, tc := range []struct { - name string - err error - }{ - {name: "success"}, - {name: "handler error", err: handlerErr}, - } { - t.Run(tc.name, func(t *testing.T) { - lease := &fakeStartupLease{} + for _, class := range []startupCommandClass{startupMutation, startupRemediation} { + for _, tc := range []struct { + name string + err error + }{ + {name: "success"}, + {name: "handler error", err: handlerErr}, + } { + t.Run(string(class)+"/"+tc.name, func(t *testing.T) { + lease := &fakeStartupLease{} + cmd := &cobra.Command{ + Use: "operation", + RunE: func(*cobra.Command, []string) error { + return tc.err + }, + } + root := &cobra.Command{Use: "root"} + registerStartupCommand(root, class, cmd) + cmd.SetContext(context.WithValue(context.Background(), startupContextKey{}, startupResult{Lease: lease})) + + err := cmd.RunE(cmd, nil) + if !errors.Is(err, tc.err) { + t.Fatalf("RunE error=%v, want %v", err, tc.err) + } + if lease.releases != 1 { + t.Fatalf("lease releases=%d, want 1", lease.releases) + } + }) + } + } +} + +func TestLeaseRetainingRegistrationJoinsLeaseReleaseError(t *testing.T) { + releaseErr := errors.New("release failed") + handlerErr := errors.New("handler failed") + for _, class := range []startupCommandClass{startupMutation, startupRemediation} { + t.Run(string(class), func(t *testing.T) { + lease := &fakeStartupLease{err: releaseErr} cmd := &cobra.Command{ - Use: "mutate", + Use: "operation", RunE: func(*cobra.Command, []string) error { - return tc.err + return handlerErr }, } root := &cobra.Command{Use: "root"} - registerStartupCommand(root, startupMutation, cmd) + registerStartupCommand(root, class, cmd) cmd.SetContext(context.WithValue(context.Background(), startupContextKey{}, startupResult{Lease: lease})) err := cmd.RunE(cmd, nil) - if !errors.Is(err, tc.err) { - t.Fatalf("RunE error=%v, want %v", err, tc.err) + if !errors.Is(err, handlerErr) { + t.Fatalf("RunE error=%v does not include handler error", err) + } + if !errors.Is(err, releaseErr) { + t.Fatalf("RunE error=%v does not include lease release error", err) } if lease.releases != 1 { t.Fatalf("lease releases=%d, want 1", lease.releases) @@ -70,32 +102,6 @@ func TestMutationRegistrationReleasesStartupLease(t *testing.T) { } } -func TestMutationRegistrationJoinsLeaseReleaseError(t *testing.T) { - releaseErr := errors.New("release failed") - handlerErr := errors.New("handler failed") - lease := &fakeStartupLease{err: releaseErr} - cmd := &cobra.Command{ - Use: "mutate", - RunE: func(*cobra.Command, []string) error { - return handlerErr - }, - } - root := &cobra.Command{Use: "root"} - registerStartupCommand(root, startupMutation, cmd) - cmd.SetContext(context.WithValue(context.Background(), startupContextKey{}, startupResult{Lease: lease})) - - err := cmd.RunE(cmd, nil) - if !errors.Is(err, handlerErr) { - t.Fatalf("RunE error=%v does not include handler error", err) - } - if !errors.Is(err, releaseErr) { - t.Fatalf("RunE error=%v does not include lease release error", err) - } - if lease.releases != 1 { - t.Fatalf("lease releases=%d, want 1", lease.releases) - } -} - func TestReadSafeRegistrationDoesNotReleaseStartupLease(t *testing.T) { lease := &fakeStartupLease{} cmd := &cobra.Command{ diff --git a/cmd/backscroll/startup_coordination.go b/cmd/backscroll/startup_coordination.go index 7d7d10f..4a62ed6 100644 --- a/cmd/backscroll/startup_coordination.go +++ b/cmd/backscroll/startup_coordination.go @@ -99,11 +99,15 @@ func coordinateStartup(ctx context.Context, cfg *config.Config, progress io.Writ } return startupLockFailure(cfg, err) } - return runOwnedStartup(ctx, cfg, progress, startupMutation, lease) + return runOwnedStartup(ctx, cfg, progress, class, lease) } } func runOwnedStartup(ctx context.Context, cfg *config.Config, progress io.Writer, class startupCommandClass, lease startupLease) startupResult { + if class == startupRemediation { + return startupResult{Config: cfg, Lease: lease} + } + // Measure index preparation time var indexPrepareStart time.Time if diagnosticsEnabled() { @@ -138,7 +142,7 @@ func runOwnedStartup(ctx context.Context, cfg *config.Config, progress io.Writer return ownedStartupFailureResult(cfg, class, lease, &startupFailure{Stage: startupStageStartupSync, Cause: err, Diagnostic: d, Recoverable: true}) } result := startupResult{Config: cfg} - if class == startupMutation { + if startupClassRetainsLease(class) { result.Lease = lease return result } @@ -147,7 +151,7 @@ func runOwnedStartup(ctx context.Context, cfg *config.Config, progress io.Writer func ownedStartupFailureResult(cfg *config.Config, class startupCommandClass, lease startupLease, failure *startupFailure) startupResult { result := startupResult{Config: cfg, Failure: failure} - if class == startupMutation { + if startupClassRetainsLease(class) { result.Lease = lease return result } diff --git a/cmd/backscroll/startup_coordination_process_test.go b/cmd/backscroll/startup_coordination_process_test.go index ef13a6a..ef83f71 100644 --- a/cmd/backscroll/startup_coordination_process_test.go +++ b/cmd/backscroll/startup_coordination_process_test.go @@ -20,7 +20,7 @@ import ( const ( startupCoordinationSeedText = "singleflight snapshot sentinel" - startupCoordinationSeedUUID = "seed-u" + startupCoordinationSeedUUID = "11111111-1111-4111-8111-111111111111" ) func TestStartupCoordinationHelperProcess(t *testing.T) { @@ -173,6 +173,48 @@ func TestStartupCoordinationMutationTimeoutNoSideEffect(t *testing.T) { assertStartupLockSidecarExists(t, dbPath) } +func TestStartupCoordinationRemediationWaitsForOwner(t *testing.T) { + dir := t.TempDir() + dbPath := filepath.Join(dir, "index.db") + seedStartupCoordinationDB(t, dbPath) + setIndexPolicyEnv(t, dbPath, t.TempDir()) + + counter := filepath.Join(dir, "sync-counter.txt") + ready := filepath.Join(dir, "owner-ready") + release := filepath.Join(dir, "owner-release") + + owner := startCoordinationChild(t, []string{"status", "--json"}, + "BACKSCROLL_SYNC_COUNTER="+counter, + "BACKSCROLL_SYNC_READY="+ready, + "BACKSCROLL_SYNC_RELEASE="+release, + "BACKSCROLL_SYNC_BLOCK=1", + ) + waitForPath(t, ready, 10*time.Second) + + blocked := startCoordinationChild(t, []string{"recover", "--from", dbPath, "--dry-run"}, + "BACKSCROLL_MUTATION_WAIT=100ms", + ) + if err := waitForChild(t, blocked, 10*time.Second); err == nil { + t.Fatal("busy remediation unexpectedly succeeded") + } + assertStderrContains(t, blocked, "sync_in_progress") + if strings.Contains(blocked.stdout.String(), "recovery dry run") { + t.Fatalf("blocked remediation emitted recovery output: %q", blocked.stdout.String()) + } + + if err := os.WriteFile(release, []byte("release"), 0o600); err != nil { + t.Fatal(err) + } + requireChildSuccess(t, owner, 10*time.Second) + + retry := startCoordinationChild(t, []string{"recover", "--from", dbPath, "--dry-run"}) + requireChildSuccess(t, retry, 10*time.Second) + if !strings.Contains(retry.stdout.String(), "recovery dry run") { + t.Fatalf("retry stdout=%q", retry.stdout.String()) + } + assertCounterLines(t, counter, 1) +} + func TestStartupCoordinationCrashRecovery(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "index.db") diff --git a/cmd/backscroll/startup_coordination_test.go b/cmd/backscroll/startup_coordination_test.go index 1139517..fa2c8ce 100644 --- a/cmd/backscroll/startup_coordination_test.go +++ b/cmd/backscroll/startup_coordination_test.go @@ -86,6 +86,32 @@ func TestCoordinateStartupImmediateOwnerMutationSyncsAndRetainsLease(t *testing. } } +func TestCoordinateStartupImmediateRemediationRetainsLeaseWithoutPrepareOrSync(t *testing.T) { + restoreStartupCoordinatorGlobals(t) + lease := &fakeStartupLease{} + startupTryAcquire = func(string) (startupLease, bool, error) { return lease, true, nil } + startupPrepareIndex = func(context.Context, *config.Config, indexCommandClass) (*storage.Database, *compat.Diagnostic, error) { + t.Fatal("remediation must not prepare the index") + return nil, nil, nil + } + startupSync = func(*config.Config, io.Writer) error { + t.Fatal("remediation must not run pre-handler sync") + return nil + } + + cfg := &config.Config{DatabasePath: filepath.Join(t.TempDir(), "index.db")} + result := coordinateStartup(context.Background(), cfg, io.Discard, startupRemediation) + if result.Failure != nil { + t.Fatalf("failure=%+v", result.Failure) + } + if result.Config != cfg || result.Lease != lease { + t.Fatalf("result=%+v want cfg and retained lease", result) + } + if lease.releases != 0 { + t.Fatalf("lease releases=%d want 0", lease.releases) + } +} + func TestCoordinateStartupBusySnapshotUsesCompatibleReadOnlySnapshot(t *testing.T) { restoreStartupCoordinatorGlobals(t) dbPath := seedCompatibleStartupDB(t) @@ -164,32 +190,65 @@ func TestCoordinateStartupBusyMutationAcquiresWithinWaitAndBecomesOwner(t *testi } } -func TestCoordinateStartupBusyMutationDeadlineReturnsSyncInProgressWithoutContinuation(t *testing.T) { +func TestCoordinateStartupBusyRemediationAcquiresAndBypassesPrepareSync(t *testing.T) { restoreStartupCoordinatorGlobals(t) - startupMutationWait = time.Millisecond + lease := &fakeStartupLease{} startupTryAcquire = func(string) (startupLease, bool, error) { return nil, false, nil } - startupAcquire = func(ctx context.Context, path string, delay time.Duration) (startupLease, error) { - <-ctx.Done() - return nil, ctx.Err() + startupAcquire = func(ctx context.Context, _ string, delay time.Duration) (startupLease, error) { + if delay != startupLockRetry { + t.Fatalf("delay=%v", delay) + } + if _, ok := ctx.Deadline(); !ok { + t.Fatal("missing deadline") + } + return lease, nil + } + startupPrepareIndex = func(context.Context, *config.Config, indexCommandClass) (*storage.Database, *compat.Diagnostic, error) { + t.Fatal("remediation must not prepare") + return nil, nil, nil } startupSync = func(*config.Config, io.Writer) error { - t.Fatal("sync should not run after mutation lock deadline") + t.Fatal("remediation must not sync") return nil } - result := coordinateStartup(context.Background(), &config.Config{DatabasePath: filepath.Join(t.TempDir(), "index.db")}, io.Discard, startupMutation) - failure := result.startupFailure() - if failure == nil { - t.Fatal("busy mutation unexpectedly succeeded") - } - if failure.Stage != startupStageSyncLock || failure.Diagnostic.Code != compat.CodeSyncInProgress { - t.Fatalf("failure=%+v want sync lock sync_in_progress", failure) + result := coordinateStartup(context.Background(), &config.Config{DatabasePath: filepath.Join(t.TempDir(), "index.db")}, io.Discard, startupRemediation) + if result.Failure != nil || result.Lease != lease { + t.Fatalf("result=%+v", result) } - if len(failure.Diagnostic.Continuation) != 0 { - t.Fatalf("continuation=%v want none", failure.Diagnostic.Continuation) + if lease.releases != 0 { + t.Fatalf("lease releases=%d want 0", lease.releases) } - if result.Lease != nil { - t.Fatalf("deadline result retained lease %+v", result.Lease) +} + +func TestCoordinateStartupBusyMutationDeadlineReturnsSyncInProgressWithoutContinuation(t *testing.T) { + for _, class := range []startupCommandClass{startupMutation, startupRemediation} { + t.Run(string(class), func(t *testing.T) { + restoreStartupCoordinatorGlobals(t) + startupMutationWait = time.Millisecond + startupTryAcquire = func(string) (startupLease, bool, error) { return nil, false, nil } + startupAcquire = func(ctx context.Context, _ string, _ time.Duration) (startupLease, error) { + <-ctx.Done() + return nil, ctx.Err() + } + startupPrepareIndex = func(context.Context, *config.Config, indexCommandClass) (*storage.Database, *compat.Diagnostic, error) { + t.Fatal("prepare after timeout") + return nil, nil, nil + } + startupSync = func(*config.Config, io.Writer) error { + t.Fatal("sync after timeout") + return nil + } + + result := coordinateStartup(context.Background(), &config.Config{DatabasePath: filepath.Join(t.TempDir(), "index.db")}, io.Discard, class) + failure := result.startupFailure() + if failure == nil || failure.Stage != startupStageSyncLock || failure.Diagnostic.Code != compat.CodeSyncInProgress || !strings.Contains(failure.Diagnostic.Summary, "retry the command") { + t.Fatalf("failure=%+v", failure) + } + if len(failure.Diagnostic.Continuation) != 0 || result.Lease != nil { + t.Fatalf("result=%+v", result) + } + }) } } diff --git a/cmd/backscroll/startup_policy.go b/cmd/backscroll/startup_policy.go index 06ae0bd..da24021 100644 --- a/cmd/backscroll/startup_policy.go +++ b/cmd/backscroll/startup_policy.go @@ -104,13 +104,6 @@ func (r startupResult) release(retErr error) error { return retErr } -func optionalStartupFailureError(f *startupFailure) error { - if f == nil { - return nil - } - return f -} - type startupContextKey struct{} var startupSync = maybeAutoSync @@ -201,9 +194,6 @@ query merges both by rank position (RRF).`, if failure == nil { return nil } - if cmd.Name() == "recover" && failure.Recoverable && class == startupMutation { - return nil - } failureErr := result.release(failure) if failure.Diagnostic.Code != "" || strings.TrimSpace(failure.Diagnostic.Summary) != "" { return refuseIndexWithCause(stdout, stderr, failure.renderedDiagnostic(), failureErr, commandBoolFlag(cmd, "json"), commandBoolFlag(cmd, "robot")) @@ -223,7 +213,7 @@ query merges both by rank position (RRF).`, registerStartupCommand(root, startupSnapshotRead, newStatusCmd(stdout, stderr)) registerStartupCommand(root, startupMetadataRead, newConfigCmd(stdout, stderr)) registerStartupCommand(root, startupMutation, newAnnotateCmd(stdout, stderr)) - registerStartupCommand(root, startupMutation, newRecoverCmd(stdout, stderr)) + registerStartupCommand(root, startupRemediation, newRecoverCmd(stdout, stderr)) return root } diff --git a/cmd/backscroll/startup_policy_test.go b/cmd/backscroll/startup_policy_test.go index 90338b6..5e74ccc 100644 --- a/cmd/backscroll/startup_policy_test.go +++ b/cmd/backscroll/startup_policy_test.go @@ -105,7 +105,7 @@ func TestEveryOperationalCommandRunsStartupBeforeHandler(t *testing.T) { {argv: []string{"status"}, wantClass: startupSnapshotRead}, {argv: []string{"validate"}, wantClass: startupSnapshotRead}, {argv: []string{"config"}, wantClass: startupMetadataRead}, - {argv: []string{"recover", "--from", "missing.db", "--dry-run"}, wantClass: startupMutation}, + {argv: []string{"recover", "--from", "missing.db", "--dry-run"}, wantClass: startupRemediation}, } for _, command := range commands { argv := command.argv @@ -146,144 +146,19 @@ func TestEveryOperationalCommandRunsStartupBeforeHandler(t *testing.T) { } } -func TestFailedStartupAllowsOnlyRecoverWithInjectedPolicy(t *testing.T) { - testCases := []struct { - name string - result startupResult - }{ - { - name: "recoverable_sync_error", - result: startupResult{ - Config: &config.Config{DatabasePath: "/tmp/error-only.db"}, - Failure: &startupFailure{ - Stage: startupStageStartupSync, - Cause: errors.New("synthetic startup error"), - Diagnostic: compat.Diagnostic{Code: compat.CodeIndexStale, Summary: "synthetic startup error", Continuation: []string{"recover", "--from", "/tmp/error-only.db", "--dry-run"}}, - Recoverable: true, - }, - }, - }, - { - name: "recoverable_diagnostic_and_error", - result: startupResult{ - Config: &config.Config{DatabasePath: "/tmp/diagnostic.db"}, - Failure: &startupFailure{ - Stage: startupStageIndexPrepare, - Diagnostic: compat.Diagnostic{ - Code: compat.CodeIndexStale, - Summary: "synthetic startup diagnostic", - Continuation: []string{"recover", "--from", "/tmp/diagnostic.db", "--dry-run"}, - }, - Cause: errors.New("synthetic startup diagnostic error"), - Recoverable: true, - }, - }, - }, - } - - blockedCommands := []struct { - name string - argv []string - }{ - {name: "search", argv: []string{"search", "needle"}}, - {name: "list", argv: []string{"list"}}, - {name: "config", argv: []string{"config"}}, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - policyCalls := 0 - policy := func(context.Context, io.Writer, startupCommandClass) startupResult { - policyCalls++ - return tc.result - } - - var recoverOut, recoverErr bytes.Buffer - recoverRoot := buildRootCmdWithStartup(&recoverOut, &recoverErr, policy) - recoverReached := false - replaceRootCommandRunE(t, recoverRoot, "recover", func(cmd *cobra.Command, args []string) error { - recoverReached = true - assertStartupResultInContext(t, startupResultFrom(cmd), tc.result) - _, _ = io.WriteString(cmd.OutOrStdout(), "recover-marker\n") - return nil - }) - recoverRoot.SetArgs([]string{"recover", "--from", "missing.db", "--dry-run"}) - if err := recoverRoot.Execute(); err != nil { - t.Fatalf("recover should proceed on startup failure: %v", err) - } - if !recoverReached { - t.Fatal("recover marker was not reached") - } - if !strings.Contains(recoverOut.String(), "recover-marker") { - t.Fatalf("recover marker output missing: stdout=%q stderr=%q", recoverOut.String(), recoverErr.String()) - } - if policyCalls != 1 { - t.Fatalf("recover startup calls=%d, want 1", policyCalls) - } - - for _, blocked := range blockedCommands { - t.Run("blocks_"+blocked.name, func(t *testing.T) { - var blockedOut, blockedErr bytes.Buffer - blockedRoot := buildRootCmdWithStartup(&blockedOut, &blockedErr, policy) - blockedReached := false - replaceRootCommandRunE(t, blockedRoot, blocked.name, func(cmd *cobra.Command, args []string) error { - blockedReached = true - _, _ = io.WriteString(cmd.OutOrStdout(), "blocked-marker\n") - return nil - }) - blockedRoot.SetArgs(blocked.argv) - err := blockedRoot.Execute() - if err == nil { - t.Fatalf("%s unexpectedly succeeded on startup failure; stdout=%q stderr=%q", blocked.name, blockedOut.String(), blockedErr.String()) - } - if blockedReached { - t.Fatalf("%s marker should not run; stdout=%q stderr=%q", blocked.name, blockedOut.String(), blockedErr.String()) - } - combined := blockedOut.String() + blockedErr.String() - if strings.Contains(combined, "blocked-marker") { - t.Fatalf("blocked command emitted marker output: stdout=%q stderr=%q", blockedOut.String(), blockedErr.String()) - } - if failure := tc.result.startupFailure(); failure != nil && failure.Diagnostic.Code != "" && !strings.Contains(blockedErr.String(), "diagnostic:") { - t.Fatalf("expected diagnostic output for blocked command; stdout=%q stderr=%q", blockedOut.String(), blockedErr.String()) - } - }) - } - if policyCalls != 1+len(blockedCommands) { - t.Fatalf("total startup calls=%d, want %d", policyCalls, 1+len(blockedCommands)) - } - }) - } -} - -func TestRecoverAloneContinuesAfterStartupFailure(t *testing.T) { - startupErr := errors.New("injected startup failure") - recoveryErr := errors.New("injected recovery failure") - called := false +func TestRemediationCommandDoesNotIgnorePolicyFailure(t *testing.T) { + policyErr := errors.New("configuration cannot be interpreted") root := buildRootCmdWithStartup(io.Discard, io.Discard, func(context.Context, io.Writer, startupCommandClass) startupResult { - dbPath := filepath.Join(t.TempDir(), "active.db") - return startupResult{Config: &config.Config{DatabasePath: dbPath}, Failure: &startupFailure{ - Stage: startupStageStartupSync, - Cause: startupErr, - Diagnostic: continuationFor(compat.Diagnostic{Code: compat.CodeIndexStale, Summary: startupErr.Error()}, dbPath), - Recoverable: true, + return startupResult{Failure: &startupFailure{ + Stage: startupStageConfigLoad, + Cause: policyErr, + Diagnostic: compat.Diagnostic{Code: compat.CodeMigrationFailed, Summary: policyErr.Error()}, }} }) - originalExecute := recoverExecute - recoverExecute = func(context.Context, recovery.Options) (recovery.Report, error) { - called = true - return recovery.Report{}, recoveryErr - } - t.Cleanup(func() { recoverExecute = originalExecute }) - root.SetArgs([]string{"recover", "--from", "stranded.db"}) + root.SetArgs([]string{"recover", "--from", "stranded.db", "--dry-run"}) err := root.Execute() - if !called { - t.Fatal("recover handler did not continue after startup failure") - } - if !errors.Is(err, startupErr) { - t.Fatalf("error=%v does not preserve startup failure", err) - } - if !errors.Is(err, recoveryErr) { - t.Fatalf("error=%v does not preserve recovery failure", err) + if !errors.Is(err, policyErr) { + t.Fatalf("error=%v want policy failure", err) } } @@ -354,46 +229,6 @@ func TestRecoverBlocksNonrecoverableStartupFailures(t *testing.T) { } } -func TestRecoverableStartupFailuresPermitControlledRecovery(t *testing.T) { - cfg := &config.Config{DatabasePath: filepath.Join(t.TempDir(), "active.db")} - for _, tc := range []struct { - name string - stage startupStage - }{ - {name: "index_prepare", stage: startupStageIndexPrepare}, - {name: "startup_sync", stage: startupStageStartupSync}, - } { - t.Run(tc.name, func(t *testing.T) { - startupDiag := continuationFor(compat.Diagnostic{Code: compat.CodeIndexStale, Summary: "recoverable " + tc.name}, cfg.DatabasePath) - called := false - originalExecute := recoverExecute - recoverExecute = func(_ context.Context, opts recovery.Options) (recovery.Report, error) { - called = true - if !opts.DryRun || opts.FromPath != cfg.DatabasePath || opts.ActivePath != cfg.DatabasePath { - t.Fatalf("recovery options=%+v, want dry-run from/active %q", opts, cfg.DatabasePath) - } - return recovery.Report{ActivePath: opts.ActivePath}, nil - } - t.Cleanup(func() { recoverExecute = originalExecute }) - - var stdout, stderr bytes.Buffer - root := buildRootCmdWithStartup(&stdout, &stderr, func(context.Context, io.Writer, startupCommandClass) startupResult { - return startupResult{Config: cfg, Failure: &startupFailure{Stage: tc.stage, Diagnostic: startupDiag, Recoverable: true}} - }) - root.SetArgs(startupDiag.Continuation) - if err := root.Execute(); err != nil { - t.Fatalf("recoverable %s did not permit dry-run recovery: %v\nstdout=%q stderr=%q", tc.stage, err, stdout.String(), stderr.String()) - } - if !called { - t.Fatal("recoverExecute was not called for recoverable startup failure") - } - if !strings.Contains(stdout.String(), "recovery dry run") { - t.Fatalf("dry-run report missing: stdout=%q stderr=%q", stdout.String(), stderr.String()) - } - }) - } -} - func TestSuccessfulStartupRecoveryFailureOmitsTypedNilStartupFailure(t *testing.T) { cfg := &config.Config{DatabasePath: filepath.Join(t.TempDir(), "active.db")} recoveryErr := errors.New("injected recovery failure") @@ -456,79 +291,6 @@ func TestSuccessfulStartupPostInstallSyncFailureOmitsTypedNilStartupFailure(t *t } } -func TestDiagnosticOnlyStartupFailurePlusRecoveryFailurePreservesBothCauses(t *testing.T) { - cfg := &config.Config{DatabasePath: filepath.Join(t.TempDir(), "active.db")} - startupDiag := continuationFor(compat.Diagnostic{Code: compat.CodeUnsupportedLineage, Summary: "diagnostic-only startup"}, cfg.DatabasePath) - recoveryErr := errors.New("injected recovery failure") - - originalExecute := recoverExecute - recoverExecute = func(context.Context, recovery.Options) (recovery.Report, error) { - return recovery.Report{}, recoveryErr - } - t.Cleanup(func() { recoverExecute = originalExecute }) - - root := buildRootCmdWithStartup(io.Discard, io.Discard, func(context.Context, io.Writer, startupCommandClass) startupResult { - return startupResult{Config: cfg, Failure: &startupFailure{Stage: startupStageIndexPrepare, Diagnostic: startupDiag, Recoverable: true}} - }) - root.SetArgs([]string{"recover", "--from", "stranded.db"}) - err := root.Execute() - if !errors.Is(err, recoveryErr) { - t.Fatalf("error=%v does not preserve recovery failure", err) - } - var failure *startupFailure - if !errors.As(err, &failure) || failure == nil { - t.Fatalf("error=%v does not expose structural startup failure", err) - } - assertDiagnosticAggregateRenderedOnce(t, err, startupDiag, recoveryErr) -} - -func TestDiagnosticOnlyStartupFailurePlusPostInstallSyncFailurePreservesBothCauses(t *testing.T) { - cfg := &config.Config{DatabasePath: filepath.Join(t.TempDir(), "active.db")} - startupDiag := continuationFor(compat.Diagnostic{Code: compat.CodeUnsupportedLineage, Summary: "diagnostic-only startup"}, cfg.DatabasePath) - syncErr := errors.New("injected post-install sync failure") - - originalExecute := recoverExecute - recoverExecute = func(context.Context, recovery.Options) (recovery.Report, error) { - return recovery.Report{ActivePath: cfg.DatabasePath}, nil - } - t.Cleanup(func() { recoverExecute = originalExecute }) - - originalPostInstallSync := recoverPostInstallSync - recoverPostInstallSync = func(*config.Config, io.Writer) error { return syncErr } - t.Cleanup(func() { recoverPostInstallSync = originalPostInstallSync }) - - var stdout bytes.Buffer - root := buildRootCmdWithStartup(&stdout, io.Discard, func(context.Context, io.Writer, startupCommandClass) startupResult { - return startupResult{Config: cfg, Failure: &startupFailure{Stage: startupStageIndexPrepare, Diagnostic: startupDiag, Recoverable: true}} - }) - root.SetArgs([]string{"recover", "--from", "stranded.db"}) - err := root.Execute() - if !errors.Is(err, syncErr) { - t.Fatalf("error=%v does not preserve post-install sync failure", err) - } - var failure *startupFailure - if !errors.As(err, &failure) || failure == nil { - t.Fatalf("error=%v does not expose structural startup failure", err) - } - assertDiagnosticAggregateRenderedOnce(t, err, startupDiag, syncErr) - if stdout.Len() != 0 { - t.Fatalf("report printed before failed post-install sync: %q", stdout.String()) - } -} - -func assertDiagnosticAggregateRenderedOnce(t *testing.T, err error, diagnostic compat.Diagnostic, cause error) { - t.Helper() - if err == nil { - t.Fatal("error is nil") - } - text := err.Error() - for _, want := range []string{string(diagnostic.Code), diagnostic.Summary, strings.Join(diagnostic.Continuation, " "), cause.Error()} { - if got := strings.Count(text, want); got != 1 { - t.Fatalf("error=%q contains %q %d times, want exactly once", text, want, got) - } - } -} - func TestStartupFailureNilAndFallbackRendering(t *testing.T) { var nilFailure *startupFailure if got := nilFailure.Error(); got != "startup failure" { @@ -883,17 +645,11 @@ func TestRejectedNonRecoverCommandReleasesBeforeDiagnostic(t *testing.T) { } } -func TestRecoverContinuationRetainsLeaseUntilHandlerReturns(t *testing.T) { +func TestRecoverRemediationRetainsLeaseUntilHandlerReturns(t *testing.T) { lease := &fakeStartupLease{} - startupErr := errors.New("startup recoverable") var stdout, stderr bytes.Buffer root := buildRootCmdWithStartup(&stdout, &stderr, func(context.Context, io.Writer, startupCommandClass) startupResult { - return startupResult{Config: &config.Config{DatabasePath: filepath.Join(t.TempDir(), "index.db")}, Lease: lease, Failure: &startupFailure{ - Stage: startupStageStartupSync, - Cause: startupErr, - Diagnostic: compat.Diagnostic{Code: compat.CodeIndexStale, Summary: startupErr.Error()}, - Recoverable: true, - }} + return startupResult{Config: &config.Config{DatabasePath: filepath.Join(t.TempDir(), "index.db")}, Lease: lease} }) replaceRootCommandRunEWrapped(t, root, "recover", func(cmd *cobra.Command, args []string) error { if lease.releases != 0 { @@ -1015,7 +771,7 @@ func replaceRootCommandRunEWrapped(t *testing.T, root *cobra.Command, commandNam for _, child := range root.Commands() { if child.Name() == commandName { child.Run = nil - child.RunE = wrapMutationRunE(runE) + child.RunE = wrapLeaseRetainingRunE(runE) return } } diff --git a/docs/adr/.stem b/docs/adr/.stem index 883a6a2..6a23db0 100644 --- a/docs/adr/.stem +++ b/docs/adr/.stem @@ -3,11 +3,11 @@ version: 2 schema: tipo: type: enum - enum: [adr] + values: [adr] required: true estado: type: enum - enum: [proposed, accepted, superseded] + values: [proposed, accepted, superseded] required: true fecha: type: string diff --git a/docs/adr/0001-declarar-frontera-documentacion-cli.md b/docs/adr/0001-declarar-frontera-documentacion-cli.md index 223b24b..d5b4252 100644 --- a/docs/adr/0001-declarar-frontera-documentacion-cli.md +++ b/docs/adr/0001-declarar-frontera-documentacion-cli.md @@ -1,7 +1,7 @@ --- tipo: adr estado: accepted -fecha: 2026-08-20 +fecha: "2026-08-20" contexto: Las guías operativas vigentes conservaron comandos y flags eliminados porque el contrato Cobra solo validaba el skill de Backscroll. decision: Mantener una lista explícita de documentación CLI vigente y validarla estáticamente contra buildRootCmd, excluyendo registros históricos declarados. consecuencias: La CI detectará deriva en ejemplos ejecutables con archivo y línea; al agregar una guía operativa deberá incorporarse a la frontera, mientras documentos históricos conservarán su contexto original. diff --git a/docs/adr/0002-exigir-sync-y-consulta-desde-sqlite.md b/docs/adr/0002-exigir-sync-y-consulta-desde-sqlite.md index 24b3deb..85f5638 100644 --- a/docs/adr/0002-exigir-sync-y-consulta-desde-sqlite.md +++ b/docs/adr/0002-exigir-sync-y-consulta-desde-sqlite.md @@ -1,7 +1,7 @@ --- tipo: adr estado: accepted -fecha: 2026-08-20 +fecha: "2026-08-20" contexto: El port Go reintrodujo el comando público read y varias rutas indexed-only que permiten consultar archivos o snapshots sin pasar por la ingesta y el índice perenne definidos por el North Star. decision: Ejecutar un sync incremental central antes de toda operación, retirar read e indexed-only y permitir únicamente recover como continuación controlada después de un intento de sync fallido. consecuencias: SQLite vuelve a ser la única fuente pública de consulta; el CLI pierde superficies incompatibles y todos los comandos operativos asumen un índice recién sincronizado. diff --git a/docs/adr/0003-coordinar-sync-entre-procesos.md b/docs/adr/0003-coordinar-sync-entre-procesos.md index 93f8196..8cfc32c 100644 --- a/docs/adr/0003-coordinar-sync-entre-procesos.md +++ b/docs/adr/0003-coordinar-sync-entre-procesos.md @@ -1,7 +1,7 @@ --- tipo: adr estado: accepted -fecha: 2026-08-21 +fecha: "2026-08-21" contexto: Varias sesiones pueden invocar Backscroll al mismo tiempo y cada proceso repite discovery, hashing, parseo y sync; el costo cuadrático al agregar texto y la serialización de escritores SQLite convierten esa duplicación en latencia y presión de recursos. decision: Eliminar la agregación completa de texto, coordinar preparación y startup sync con un lock advisory del sistema operativo por base canónica y permitir que los procesos read-safe no propietarios consulten una snapshot confirmada compatible. consecuencias: Habrá un solo owner de preparación, sync y mutaciones por base; las lecturas concurrentes conservarán disponibilidad a cambio de observar temporalmente una snapshot anterior y de introducir un sidecar persistente, diagnósticos de frescura y una dependencia de locking portable. diff --git a/docs/adr/0004-identidad-semantica-y-cierre-de-linajes.md b/docs/adr/0004-identidad-semantica-y-cierre-de-linajes.md new file mode 100644 index 0000000..f7ee059 --- /dev/null +++ b/docs/adr/0004-identidad-semantica-y-cierre-de-linajes.md @@ -0,0 +1,83 @@ +--- +tipo: adr +estado: accepted +fecha: "2026-08-24" +contexto: Backscroll ha rechazado repetidamente bases SQLite sanas porque la identidad de compatibilidad mezcla semántica actual, formato textual del DDL y checksums históricos; V14 volvió a bloquear dos linajes V13 reconocidos y la prueba de migración los ocultó mediante skips. +decision: Identificar los esquemas mediante una forma semántica híbrida, conservar el ledger como proveniencia separada y exigir que cada fixture histórica reconocida desde V1 migre hasta head mediante la ruta productiva sin exclusiones; recover será una ruta de remediación que no depende de la preparación ordinaria. +consecuencias: Las diferencias cosméticas y las historias equivalentes dejarán de multiplicar linajes, toda migración futura probará automáticamente el corpus histórico completo y los esquemas semánticamente desconocidos seguirán fallando de forma cerrada; aumentará la responsabilidad del canonicalizador y de las pruebas de no equivalencia. +--- + +# Adoptar identidad semántica y cierre completo de linajes + +## Contexto + +La compatibilidad de índices se introdujo para reconocer la forma real de una base SQLite y no confiar solamente en su versión de migración. La implementación actual firma metadatos estructurados, texto DDL normalizado y filas de `schema_migrations`, y consulta un catálogo hermético de releases y formas observadas. + +Este modelo produjo bloqueos recurrentes. Los issues #41 y #52 incorporaron formas V13 faltantes y corrigieron diferencias de whitespace, un catálogo duplicado y defectos léxicos. Sin embargo, V14 preservó diferencias textuales heredadas por tablas construidas mediante `ALTER TABLE`. Dos formas V13 ya reconocidas produjeron firmas V14 desconocidas; la verificación anterior al commit abortó y todos los comandos operacionales quedaron bloqueados. + +La especificación sistémica previa exigía que cada linaje publicado alcanzara head sin pérdida y sin pruebas omitidas. La implementación contradijo esa garantía al excluir expresamente las dos fixtures ALTER-built de la prueba de migración. El defecto estaba presente en el corpus, pero la suite fue configurada para no ejecutarlo. + +La causa no se limita a V13. Cualquier forma histórica reconocida desde V1 puede conservar diferencias de construcción a través de migraciones posteriores. Añadir firmas V14 resolvería el bloqueo inmediato, pero permitiría que una migración futura repitiera la multiplicación. + +También se verificó que `recover --from … --dry-run` no remedia este fallo: la política de startup intenta primero la misma preparación incompatible y vuelve a emitir `migration_failed`. + +## Decisión + +La identidad de compatibilidad se dividirá en dos conceptos: + +1. **Forma semántica actual.** Será la clave para seleccionar un plan. Combinará metadatos estructurados de SQLite con SQL auxiliar canonicalizado para constraints, expresiones, triggers, índices parciales y tablas virtuales que PRAGMA no describe completamente. +2. **Proveniencia de migración.** Las filas, nombres y checksums de `schema_migrations` seguirán inspeccionándose, preservándose y documentándose, pero no crearán identidades distintas cuando la versión aplicada y la forma semántica sean equivalentes. + +El canonicalizador eliminará comentarios y diferencias irrelevantes de whitespace o puntuación fuera de literales e identificadores citados. Preservará strings, identificadores citados y diferencias conductuales. No incorporará un parser SQL completo. + +El catálogo continuará siendo la fuente hermética de releases y formas observadas. Varias fixtures físicas podrán compartir firma semántica, pero cada fixture permanecerá como entrada independiente de prueba. Una colisión será válida únicamente si coincide la versión aplicada, la evidencia estructural, la semántica auxiliar y el plan restante. + +La prueba de cierre se derivará automáticamente de todas las fixtures. Para cada forma reconocida desde V1 ejecutará `OpenCompatible`, aplicará todas las migraciones reales hasta head y verificará forma final, filas, UUIDs, satélites, FTS, ledger y snapshots aplicables. No tendrá allowlist manual ni rama de `t.Skip`. Añadir una migración futura extenderá automáticamente todas las trayectorias históricas al nuevo head. + +`recover` se clasificará como remediación. Conservará el lock exclusivo, pero omitirá `OpenCompatible` y el sync previo al handler. Su planner abrirá los inputs mediante rutas read-only y, en apply, conservará el lock durante reemplazo, verificación y post-install sync. `--dry-run` no modificará datos de entrada. + +Los esquemas cuya forma semántica no figure en el catálogo seguirán devolviendo `unsupported_lineage`. No se añadirán flags de bypass, catálogos paralelos, reparación automática ni excepciones runtime para V14. + +## Alternativas descartadas + +### Registrar las firmas V14 faltantes + +Desbloquea los casos actuales con poco código, pero conserva la multiplicación de identidades por historia y formato. + +### Ajustar solamente whitespace y puntuación + +Corrige el mecanismo inmediato, pero mantiene el ledger histórico y otras diferencias textuales dentro de la identidad actual. + +### Construir un parser DDL completo + +Podría producir una representación AST más profunda, pero añade un subsistema complejo para dialecto, triggers, virtual tables y extensiones futuras. El modelo híbrido prioriza metadatos de SQLite. + +### Confiar solamente en versión y checksum + +Versiones iguales ya han ocultado formas distintas, mientras checksums distintos pueden terminar en la misma forma actual. Esta alternativa confunde proveniencia con compatibilidad. + +### Mantener recover en la preparación ordinaria + +Obliga al comando reparador a superar primero la condición que debe reparar y produce una continuación operacionalmente circular. + +## Consecuencias + +### Positivas + +- Las diferencias cosméticas y las rutas equivalentes dejan de multiplicar identidades. +- Cada release y forma observada permanece representada por una fixture verificable. +- Toda migración futura prueba automáticamente cada trayectoria histórica desde V1. +- La suite no puede declarar soportada una forma y excluirla simultáneamente. +- Los usuarios ALTER-built alcanzarán head sin editar `sqlite_master`. +- `recover --dry-run` podrá alcanzar su planner aunque la preparación ordinaria falle. + +### Negativas + +- El canonicalizador auxiliar será una frontera de seguridad con pruebas exigentes. +- Regenerar el catálogo cambiará firmas aunque los bytes de fixtures no cambien. +- La matriz histórica aumentará el tiempo de pruebas de storage. +- Separar proveniencia e identidad exige aclarar APIs cuyo campo se denomina `Signature`. + +### Riesgo aceptado + +La canonicalización será conservadora: puede mantener separadas algunas formas equivalentes no demostradas por fixtures, pero no debe fusionar formas con comportamiento distinto. Se aceptan falsos negativos documentables para evitar falsos positivos inseguros. diff --git a/docs/adr/0005-consultar-catalogo-por-forma-completa.md b/docs/adr/0005-consultar-catalogo-por-forma-completa.md new file mode 100644 index 0000000..82457bd --- /dev/null +++ b/docs/adr/0005-consultar-catalogo-por-forma-completa.md @@ -0,0 +1,43 @@ +--- +tipo: adr +estado: accepted +fecha: "2026-08-24" +contexto: La firma semántica dejó de incluir las filas históricas de schema_migrations. Como consecuencia, varias bases pueden compartir la misma firma de forma actual aunque tengan distinta versión aplicada y, por lo tanto, distinto plan de migración restante. +decision: Consultar el catálogo de linajes mediante la forma completa compuesta por AppliedVersion y Signature. La proveniencia del ledger se inspecciona y valida de manera privada, pero no participa en la firma semántica ni se expone en SchemaShape. +consecuencias: El catálogo puede representar firmas compartidas sin ambigüedad, recovery rechaza una firma conocida con versión desconocida y las APIs de solo firma se eliminan; los llamadores deben transportar SchemaShape completo en lugar de strings de firma aislados. +--- + +# Consultar catálogo de linajes por forma completa + +## Contexto + +La separación entre identidad semántica y proveniencia de migración evita que dos bases con la misma forma actual queden en linajes diferentes solo por checksums, nombres o relojes históricos del ledger. Esa separación también permite que dos versiones aplicadas compartan una misma firma semántica. + +Con una consulta basada únicamente en `Signature`, el catálogo tendría que escoger un único ganador para firmas compartidas. Ese ganador podría tener un `AppliedVersion` distinto al observado y producir pasos de migración incorrectos. Recovery presentaba el mismo riesgo: podía aceptar una firma conocida aunque la versión aplicada no perteneciera al catálogo. + +## Decisión + +La identidad operacional de catálogo será `SchemaShape{AppliedVersion, Signature}`. Internamente se representa con una clave compuesta `lineageKey` y se exponen las operaciones `ByShape`, `IsKnownShape` y `CurrentShape`. + +Las filas de `schema_migrations` se cargan como proveniencia privada, se validan en orden y determinan `AppliedVersion`, pero no se agregan a los registros semánticos que producen `Signature`. Recovery valida la forma completa recibida desde el plan de compatibilidad antes de leer registros recuperables. + +## Alternativas descartadas + +### Mantener consulta solo por firma + +Se descartó porque una firma compartida entre versiones aplicadas distintas vuelve ambigua la selección de `remainingSteps` y puede aceptar bases no catalogadas. + +### Incluir la versión aplicada dentro de la firma + +Se descartó porque volvería a mezclar identidad semántica con estado de migración. La firma debe describir la forma actual; la versión aplicada debe participar en la identidad de catálogo como dimensión separada. + +### Exponer proveniencia completa en `SchemaShape` + +Se descartó porque los nombres y checksums del ledger son evidencia histórica, no contrato público de compatibilidad ni entrada necesaria para planes de migración. + +## Consecuencias + +- Varias fixtures físicas pueden compartir firma sin colisionar en el catálogo. +- Los llamadores deben conservar y pasar `SchemaShape` completo. +- `BySignature`, `IsKnownSignature` y `CurrentSignature` dejan de existir. +- Los diagnósticos de linaje no soportado deben incluir versión y firma para explicar la identidad rechazada. diff --git a/docs/adr/0006-seleccionar-cabeza-semantica-por-version-maxima.md b/docs/adr/0006-seleccionar-cabeza-semantica-por-version-maxima.md new file mode 100644 index 0000000..612240f --- /dev/null +++ b/docs/adr/0006-seleccionar-cabeza-semantica-por-version-maxima.md @@ -0,0 +1,48 @@ +--- +tipo: adr +estado: accepted +fecha: "2026-08-24" +contexto: El catálogo puede contener varias firmas semánticas legítimas para versiones históricas, como V3 y V5, mientras que la cabeza semántica puede provenir de una fixture no manifestada con AppliedVersion mayor que LatestGoRelease. +decision: Seleccionar CurrentShape mediante la máxima AppliedVersion inventariada y validar ambigüedad solo entre las formas que comparten esa versión máxima. LatestGoRelease conserva su rol de validación de inventario, pero no decide la cabeza semántica. +consecuencias: Las firmas históricas múltiples dejan de producir fallos dependientes del orden de iteración de mapas; una competencia real en la versión máxima sigue fallando de forma cerrada; las pruebas de cierre comparan cero pasos contra la forma semántica corriente real. +--- + +# Seleccionar cabeza semántica por versión máxima + +## Contexto + +El catálogo de compatibilidad representa evidencias físicas de releases publicadas y fixtures locales no manifestadas. Algunas versiones históricas tienen más de una firma semántica legítima porque las bases observadas pueden diferir en columnas fantasma o metadatos de origen y aun así conservar planes de migración válidos. + +La selección previa de cabeza dependía del release Go más reciente. Durante la transición a identidad semántica compuesta, una implementación parcial intentó calcular la versión máxima y validar firmas en una sola pasada sobre un `map`. Ese enfoque podía tratar temporalmente firmas V3 o V5 como cabeza corriente antes de observar V14, lo que introducía fallos intermitentes por orden de iteración. + +## Decisión + +`Catalog.attachLineages` seleccionará `CurrentShape()` en dos pasos: + +1. calcular la máxima `AppliedVersion` entre todos los linajes inventariados; +2. revisar únicamente los linajes con esa versión máxima. + +Si todos los linajes de la versión máxima tienen la misma firma semántica, esa forma se considera cabeza corriente. Si existen firmas distintas en la versión máxima, el catálogo falla cerrado con error de ambigüedad. + +`LatestGoRelease` permanece como validación de inventario en la carga del manifiesto: debe existir, cumplir el piso de versión esperado y conservar fixtures verificables. No participa en la selección de cabeza semántica. + +## Alternativas descartadas + +### Usar `LatestGoRelease` como fuente de verdad de cabeza + +Se descartó porque la cabeza semántica puede estar representada por una fixture local no manifestada con mayor `AppliedVersion`. En ese caso, usar el release Go más reciente degradaría la cabeza a una versión anterior. + +### Aceptar cualquier firma cuando no quedan pasos + +Se descartó porque relajaría la frontera de compatibilidad. Una forma corriente ambigua en la máxima versión debe detener la carga para evitar aceptar estados no catalogados. + +### Agregar aliases o fixtures `*-current.sql` + +Se descartó porque las fixtures ALTER-built ya son estructuralmente equivalentes a la cabeza canónica salvo formato no semántico. Crear aliases encubriría el problema de selección en lugar de corregir la regla del catálogo. + +## Consecuencias + +- Las firmas múltiples en versiones históricas no-cabeza son compatibles con el catálogo. +- La selección de cabeza ya no depende del orden no determinista de iteración de mapas en Go. +- Una divergencia real entre firmas de la máxima `AppliedVersion` sigue siendo un error fatal de catálogo. +- Las pruebas de cierre pueden exigir simultáneamente cero pasos restantes y `plan.From == catalog.CurrentShape()` para cada fixture física. diff --git a/docs/superpowers/plans/2026-08-24-recover-remediation-startup.md b/docs/superpowers/plans/2026-08-24-recover-remediation-startup.md new file mode 100644 index 0000000..91d954a --- /dev/null +++ b/docs/superpowers/plans/2026-08-24-recover-remediation-startup.md @@ -0,0 +1,603 @@ +# Recover Remediation Startup Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make `recover` retain exclusive startup-lock ownership while bypassing ordinary compatible-open and pre-handler sync so remediation remains reachable when normal index preparation fails. + +**Architecture:** Add one explicit `startupRemediation` command class beside snapshot, metadata, and mutation classes. It follows mutation lock acquisition/timeout and handler-held lease semantics, but `runOwnedStartup` returns directly after lock acquisition without calling `prepareIndex` or `maybeAutoSync`. Recovery's existing handler remains responsible for read-only planning, atomic replacement, verification, and post-install sync. + +**Tech Stack:** Go 1.26.2, Cobra, `gofrs/flock`, SQLite recovery package, stdlib testing, subprocess tests, Just + +**Spec:** `docs/superpowers/specs/2026-08-24-semantic-lineage-closure-design.md` + +**Dependency:** Execute after `docs/superpowers/plans/2026-08-24-semantic-lineage-closure.md`, so the ALTER-built fixture is semantically recognized by recovery adapters. + +## Global Constraints + +- `recover` keeps the canonical database startup lock for the complete handler. +- Remediation uses the same five-second mutation wait and busy-owner timeout as mutations. +- Remediation skips ordinary `OpenCompatible` preparation and mandatory pre-handler sync. +- Invalid CLI/config/manifest input still fails before recovery planning. +- `recover --dry-run` performs no input-data mutation. +- Apply mode keeps the lease through atomic replacement, verification, and post-install sync. +- Search, list, patterns, status, validate, config, annotate, purge, and rebuild behavior is unchanged. +- Remove the old mechanism that carries an ordinary startup failure into the recover handler. +- Do not add a new command, flag, lock file, retry loop, or persisted state. +- Follow RED → GREEN → REFACTOR; run focused tests before race and repository gates. + +--- + +## File map + +| Path | Responsibility | +|---|---| +| `cmd/backscroll/startup_commands.go` | Declare remediation class and shared lease-retaining class predicate. | +| `cmd/backscroll/startup_commands_test.go` | Explicit command classification and lease-release behavior. | +| `cmd/backscroll/startup_coordination.go` | Mutation-equivalent lock acquisition with remediation preparation bypass. | +| `cmd/backscroll/startup_coordination_test.go` | Immediate/busy owner bypass, timeout, and lease retention. | +| `cmd/backscroll/startup_policy.go` | Register recover as remediation and remove startup-failure pass-through exception. | +| `cmd/backscroll/startup_policy_test.go` | Validate preflight blocking and delete obsolete controlled-failure path tests. | +| `cmd/backscroll/recover.go` | Consume successful remediation startup only; retain post-install sync. | +| `cmd/backscroll/recover_test.go` | Handler error and lease release without startup-failure aggregation. | +| `cmd/backscroll/compat_diagnostics_test.go` | Real ALTER-built dry-run reaches planner and preserves SQLite files. | +| `cmd/backscroll/startup_coordination_process_test.go` | Cross-process remediation contention and lock retention. | +| `docs/sync.md` | Document remediation owner path. | +| `CLAUDE.md` | Update startup command classes and recover flow. | + +### Task 1: Add an explicit lease-retaining remediation class + +**Files:** +- Modify: `cmd/backscroll/startup_commands.go:5-43` +- Modify: `cmd/backscroll/startup_commands_test.go:10-128` +- Modify: `cmd/backscroll/startup_coordination.go:140-153` +- Modify: `cmd/backscroll/startup_policy.go:180-224` +- Modify: `cmd/backscroll/startup_policy_test.go:93-145` + +**Interfaces:** +- Consumes: existing Cobra annotations and `startupResult.release`. +- Produces: + +```go +const startupRemediation startupCommandClass = "remediation" +func startupClassRetainsLease(startupCommandClass) bool +``` + +- [ ] **Step 1: Write the command-class RED test** + +Change `TestEveryOperationalCommandHasApprovedStartupClass` expectations: + +```go +"recover": startupRemediation, +``` + +Change `TestEveryOperationalCommandRunsStartupBeforeHandler` similarly: + +```go +{argv: []string{"recover", "--from", "missing.db", "--dry-run"}, wantClass: startupRemediation}, +``` + +Run: + +```bash +go test ./cmd/backscroll -run '^(TestEveryOperationalCommandHasApprovedStartupClass|TestEveryOperationalCommandRunsStartupBeforeHandler)$' +``` + +Expected: FAIL to compile with `undefined: startupRemediation`. + +- [ ] **Step 2: Add remediation to the explicit class set** + +In `startup_commands.go`: + +```go +const ( + startupSnapshotRead startupCommandClass = "snapshot-read" + startupMetadataRead startupCommandClass = "metadata-read" + startupMutation startupCommandClass = "mutation" + startupRemediation startupCommandClass = "remediation" + startupClassKey = "backscroll.io/startup-class" +) +``` + +Accept it in `startupCommandClassFor`: + +```go +case startupSnapshotRead, startupMetadataRead, startupMutation, startupRemediation: +``` + +Add: + +```go +func startupClassRetainsLease(class startupCommandClass) bool { + return class == startupMutation || class == startupRemediation +} +``` + +Use the predicate in registration: + +```go +if startupClassRetainsLease(class) { + cmd.RunE = wrapLeaseRetainingRunE(cmd.RunE) +} +``` + +Rename `wrapMutationRunE` to `wrapLeaseRetainingRunE`; its body remains the same. + +- [ ] **Step 3: Register recover as remediation** + +In `buildRootCmdWithStartup`: + +```go +registerStartupCommand(root, startupRemediation, newRecoverCmd(stdout, stderr)) +``` + +Do not change another command's class. + +- [ ] **Step 4: Preserve existing startup behavior while the new class is introduced** + +In `runOwnedStartup` and `ownedStartupFailureResult`, replace `class == startupMutation` with `startupClassRetainsLease(class)`. Do not bypass prepare/sync yet. + +Temporarily update the existing recover-on-recoverable-failure condition to use the same predicate: + +```go +if cmd.Name() == "recover" && failure.Recoverable && startupClassRetainsLease(class) { + return nil +} +``` + +Task 3 deletes this compatibility bridge after Task 2 gives remediation its direct owner path. This keeps the complete `cmd/backscroll` suite green between commits. + +- [ ] **Step 5: Generalize lease-release registration tests** + +Rename `TestMutationRegistrationReleasesStartupLease` to `TestLeaseRetainingRegistrationReleasesStartupLease` and table-drive both classes and handler outcomes: + +```go +handlerErr := errors.New("handler failed") +for _, class := range []startupCommandClass{startupMutation, startupRemediation} { + for _, tc := range []struct{name string; err error}{{name: "success"}, {name: "handler error", err: handlerErr}} { + t.Run(string(class)+"/"+tc.name, func(t *testing.T) { + lease := &fakeStartupLease{} + cmd := &cobra.Command{Use: "operation", RunE: func(*cobra.Command, []string) error { return tc.err }} + root := &cobra.Command{Use: "root"} + registerStartupCommand(root, class, cmd) + cmd.SetContext(context.WithValue(context.Background(), startupContextKey{}, startupResult{Lease: lease})) + err := cmd.RunE(cmd, nil) + if !errors.Is(err, tc.err) { t.Fatalf("error=%v want %v", err, tc.err) } + if lease.releases != 1 { t.Fatalf("releases=%d want 1", lease.releases) } + }) + } +} +``` + +For the release-error join test, loop over the same two classes, use `fakeStartupLease{err: releaseErr}`, and assert `errors.Is(err, handlerErr)`, `errors.Is(err, releaseErr)`, and exactly one release. Keep the read-safe no-release test unchanged. + +- [ ] **Step 6: Run focused and complete command tests to GREEN** + +```bash +go test ./cmd/backscroll -run '^(TestEveryOperationalCommandHasApprovedStartupClass|TestEveryOperationalCommandRunsStartupBeforeHandler|TestLeaseRetainingRegistration|TestReadSafeRegistration|TestUnknownStartupCommandClass)' +go test ./cmd/backscroll +``` + +Expected: PASS. Recovery still performs ordinary preparation at this intermediate commit; only its explicit class and lease semantics have changed. + +- [ ] **Step 7: Commit the explicit command contract** + +```bash +git add cmd/backscroll/startup_commands.go cmd/backscroll/startup_commands_test.go \ + cmd/backscroll/startup_coordination.go cmd/backscroll/startup_policy.go cmd/backscroll/startup_policy_test.go +git commit -m "refactor(cli): classify recover as remediation" +``` + +### Task 2: Bypass prepare and sync after remediation acquires the lock + +**Files:** +- Modify: `cmd/backscroll/startup_coordination.go:31-153` +- Modify: `cmd/backscroll/startup_coordination_test.go:17-260` + +**Interfaces:** +- Consumes: `startupRemediation` and `startupClassRetainsLease` from Task 1. +- Produces: owned remediation `startupResult{Config: cfg, Lease: lease}` with zero prepare/sync calls. + +- [ ] **Step 1: Write immediate-owner bypass test** + +Add: + +```go +func TestCoordinateStartupImmediateRemediationRetainsLeaseWithoutPrepareOrSync(t *testing.T) { + restoreStartupCoordinatorGlobals(t) + lease := &fakeStartupLease{} + startupTryAcquire = func(string) (startupLease, bool, error) { return lease, true, nil } + startupPrepareIndex = func(context.Context, *config.Config, indexCommandClass) (*storage.Database, *compat.Diagnostic, error) { + t.Fatal("remediation must not prepare the index") + return nil, nil, nil + } + startupSync = func(*config.Config, io.Writer) error { + t.Fatal("remediation must not run pre-handler sync") + return nil + } + + cfg := &config.Config{DatabasePath: filepath.Join(t.TempDir(), "index.db")} + result := coordinateStartup(context.Background(), cfg, io.Discard, startupRemediation) + if result.Failure != nil { t.Fatalf("failure=%+v", result.Failure) } + if result.Config != cfg || result.Lease != lease { t.Fatalf("result=%+v want cfg and retained lease", result) } + if lease.releases != 0 { t.Fatalf("lease releases=%d want 0", lease.releases) } +} +``` + +- [ ] **Step 2: Write busy-owner remediation test** + +Add: + +```go +func TestCoordinateStartupBusyRemediationAcquiresAndBypassesPrepareSync(t *testing.T) { + restoreStartupCoordinatorGlobals(t) + lease := &fakeStartupLease{} + startupTryAcquire = func(string) (startupLease, bool, error) { return nil, false, nil } + startupAcquire = func(ctx context.Context, _ string, delay time.Duration) (startupLease, error) { + if delay != startupLockRetry { t.Fatalf("delay=%v", delay) } + if _, ok := ctx.Deadline(); !ok { t.Fatal("missing deadline") } + return lease, nil + } + startupPrepareIndex = func(context.Context, *config.Config, indexCommandClass) (*storage.Database, *compat.Diagnostic, error) { + t.Fatal("remediation must not prepare") + return nil, nil, nil + } + startupSync = func(*config.Config, io.Writer) error { t.Fatal("remediation must not sync"); return nil } + result := coordinateStartup(context.Background(), &config.Config{DatabasePath: filepath.Join(t.TempDir(), "index.db")}, io.Discard, startupRemediation) + if result.Failure != nil || result.Lease != lease { t.Fatalf("result=%+v", result) } +} +``` + +Run: + +```bash +go test ./cmd/backscroll -run '^TestCoordinateStartup.*Remediation' +``` + +Expected: both tests FAIL because `runOwnedStartup` prepares and syncs; the busy path also changes the class to `startupMutation`. + +- [ ] **Step 3: Preserve the requested class after waiting** + +In `coordinateStartup`, change: + +```go +return runOwnedStartup(ctx, cfg, progress, startupMutation, lease) +``` + +to: + +```go +return runOwnedStartup(ctx, cfg, progress, class, lease) +``` + +The default switch branch continues to provide mutation/remediation wait semantics. Snapshot and metadata branches remain unchanged. + +- [ ] **Step 4: Return remediation ownership before ordinary preparation** + +At the beginning of `runOwnedStartup`, before diagnostics timing or `startupPrepareIndex`: + +```go +if class == startupRemediation { + return startupResult{Config: cfg, Lease: lease} +} +``` + +Task 1 already generalized the normal post-sync and failure retention paths through `startupClassRetainsLease`; do not add a second class check. + +- [ ] **Step 5: Table-drive contention timeout for mutation and remediation** + +Change the busy deadline test to run: + +```go +for _, class := range []startupCommandClass{startupMutation, startupRemediation} { + t.Run(string(class), func(t *testing.T) { + restoreStartupCoordinatorGlobals(t) + startupMutationWait = time.Millisecond + startupTryAcquire = func(string) (startupLease, bool, error) { return nil, false, nil } + startupAcquire = func(ctx context.Context, _ string, _ time.Duration) (startupLease, error) { <-ctx.Done(); return nil, ctx.Err() } + startupPrepareIndex = func(context.Context, *config.Config, indexCommandClass) (*storage.Database, *compat.Diagnostic, error) { t.Fatal("prepare after timeout"); return nil, nil, nil } + startupSync = func(*config.Config, io.Writer) error { t.Fatal("sync after timeout"); return nil } + result := coordinateStartup(context.Background(), &config.Config{DatabasePath: filepath.Join(t.TempDir(), "index.db")}, io.Discard, class) + failure := result.startupFailure() + if failure == nil || failure.Diagnostic.Code != compat.CodeSyncInProgress { t.Fatalf("failure=%+v", failure) } + if len(failure.Diagnostic.Continuation) != 0 || result.Lease != nil { t.Fatalf("result=%+v", result) } + }) +} +``` + +- [ ] **Step 6: Run coordination tests** + +```bash +go test ./cmd/backscroll -run '^TestCoordinateStartup' -count=1 +``` + +Expected: PASS. + +- [ ] **Step 7: Commit the remediation owner path** + +```bash +git add cmd/backscroll/startup_coordination.go cmd/backscroll/startup_coordination_test.go +git commit -m "fix(cli): bypass ordinary startup for recovery" +``` + +### Task 3: Remove startup-failure pass-through from the recover handler + +**Files:** +- Modify: `cmd/backscroll/startup_policy.go:96-111,192-216` +- Modify: `cmd/backscroll/startup_policy_test.go:149-518` +- Modify: `cmd/backscroll/recover.go:30-72` +- Modify: `cmd/backscroll/recover_test.go:160-275` +- Modify: `cmd/backscroll/compat_diagnostics_test.go:540-600` + +**Interfaces:** +- Consumes: successful remediation startup results from Task 2. +- Produces: recovery errors and post-install sync errors without aggregation with an ordinary startup failure. + +- [ ] **Step 1: Write a policy test proving remediation does not bypass genuine startup failures** + +Add: + +```go +func TestRemediationCommandDoesNotIgnorePolicyFailure(t *testing.T) { + policyErr := errors.New("configuration cannot be interpreted") + root := buildRootCmdWithStartup(io.Discard, io.Discard, func(context.Context, io.Writer, startupCommandClass) startupResult { + return startupResult{Failure: &startupFailure{ + Stage: startupStageConfigLoad, + Cause: policyErr, + Diagnostic: compat.Diagnostic{Code: compat.CodeMigrationFailed, Summary: policyErr.Error()}, + }} + }) + root.SetArgs([]string{"recover", "--from", "stranded.db", "--dry-run"}) + err := root.Execute() + if !errors.Is(err, policyErr) { t.Fatalf("error=%v want policy failure", err) } +} +``` + +This should already pass; it locks the retained preflight boundary before deleting the special case. + +- [ ] **Step 2: Delete the old recover-on-failed-startup exception** + +Remove from `PersistentPreRunE`: + +```go +if cmd.Name() == "recover" && failure.Recoverable && startupClassRetainsLease(class) { + return nil +} +``` + +Every startup failure now follows the ordinary refusal path. Real remediation never creates prepare/sync failure because it bypasses those phases. + +- [ ] **Step 3: Simplify the recover handler** + +Delete `optionalStartupFailureError` from `startup_policy.go`. + +In `recover.go`, delete: + +```go +startupFailure := optionalStartupFailureError(startup.startupFailure()) +``` + +Return direct wrapped errors: + +```go +return fmt.Errorf("load config for recovery: %w", err) +return fmt.Errorf("recovery failed: %w", err) +return fmt.Errorf("post-recovery sync: %w", err) +``` + +Do not join an absent startup failure. Keep backup-path and installed-path stderr reporting unchanged. + +- [ ] **Step 4: Remove tests for the obsolete aggregate path** + +Delete or replace tests whose sole contract is that recover continues with a failed ordinary startup result: + +- `TestFailedStartupAllowsOnlyRecoverWithInjectedPolicy` +- `TestRecoverAloneContinuesAfterStartupFailure` +- `TestRecoverableStartupFailuresPermitControlledRecovery` +- `TestDiagnosticOnlyStartupFailurePlusRecoveryFailurePreservesBothCauses` +- `TestDiagnosticOnlyStartupFailurePlusPostInstallSyncFailurePreservesBothCauses` + +Keep tests for invalid invocation, nonrecoverable preflight failure, normal recovery errors, backup reporting, post-install sync, and machine diagnostics. + +Update recover unit tests that inject `startupResult.Failure`: inject only `Config` and expected handler failures. Assert no typed `startupFailure` is present in returned errors. + +- [ ] **Step 5: Replace injected continuation success with the real default policy** + +Rewrite `TestRecoveryContinuationExecutesInConfiguredSamePathContextWithEmptyWAL` as `TestRecoverDryRunBypassesPreparationForAlterBuiltLineage`. + +Use existing helpers: + +```go +dbPath := newFixtureIndexDB(t, "v13-development-alter-built.sql") +setIndexPolicyEnv(t, dbPath, t.TempDir()) +``` + +Create the empty WAL and call `snapshotSQLiteFiles` as the current test does. Execute the real command: + +```go +stdout, stderr, err := runCmd("recover", "--from", dbPath, "--dry-run") +if err != nil { + t.Fatalf("recover dry-run failed: %v\nstdout=%q stderr=%q", err, stdout, stderr) +} +``` + +Assert: + +```go +if !strings.Contains(stdout, "recovery dry run") { t.Fatalf("stdout=%q", stdout) } +for _, forbidden := range []string{"migration_failed", "unsupported_lineage", "f6a081b9", "50016 diagnostic"} { + if strings.Contains(stdout+stderr, forbidden) { t.Fatalf("output retained %q: stdout=%q stderr=%q", forbidden, stdout, stderr) } +} +``` + +Retain exact database/WAL metadata immutability assertions, including: + +```go +assertSQLiteFilesUnchanged(t, dbPath, before) +walAfter, err := os.Stat(walPath) +if err != nil { t.Fatalf("stat WAL after dry-run: %v", err) } +if walAfter.Size() != 0 || walAfter.Mode() != walBefore.Mode() || !walAfter.ModTime().Equal(walBefore.ModTime()) { + t.Fatalf("empty WAL metadata changed: before=%+v after=%+v", walBefore, walAfter) +} +``` + +- [ ] **Step 6: Run all command tests after removing the compatibility bridge** + +```bash +go test ./cmd/backscroll -run '^(TestRemediationCommandDoesNotIgnorePolicyFailure|TestInvalidOperationalCommandsSkipStartup|TestRecover|TestSuccessfulStartup|TestRecoverDryRunBypassesPreparationForAlterBuiltLineage)' -count=1 +go test ./cmd/backscroll +``` + +Expected: PASS with a recovery dry-run report and no repeated startup diagnostic. The complete package must be green before commit. + +- [ ] **Step 7: Commit removal of the circular failure channel** + +```bash +git add cmd/backscroll/startup_policy.go cmd/backscroll/startup_policy_test.go \ + cmd/backscroll/recover.go cmd/backscroll/recover_test.go cmd/backscroll/compat_diagnostics_test.go +git commit -m "refactor(recovery): remove startup failure pass-through" +``` + +### Task 4: Prove remediation contention across processes + +**Files:** +- Modify: `cmd/backscroll/startup_coordination_process_test.go` + +**Interfaces:** +- Consumes: remediation startup from Tasks 1–3 and existing subprocess barriers. +- Produces: cross-process wait, timeout, retry, and zero-pre-sync evidence. + +- [ ] **Step 1: Add a process test proving remediation waits and retains ownership** + +Add `TestStartupCoordinationRemediationWaitsForOwner` using the existing child helpers and sync barrier—no new helper protocol is needed: + +```go +dir := t.TempDir() +dbPath := filepath.Join(dir, "index.db") +seedStartupCoordinationDB(t, dbPath) +setIndexPolicyEnv(t, dbPath, t.TempDir()) +counter := filepath.Join(dir, "sync-counter.txt") +ready := filepath.Join(dir, "owner-ready") +release := filepath.Join(dir, "owner-release") + +owner := startCoordinationChild(t, []string{"status", "--json"}, + "BACKSCROLL_SYNC_COUNTER="+counter, + "BACKSCROLL_SYNC_READY="+ready, + "BACKSCROLL_SYNC_RELEASE="+release, + "BACKSCROLL_SYNC_BLOCK=1") +waitForPath(t, ready, 10*time.Second) + +blocked := startCoordinationChild(t, []string{"recover", "--from", dbPath, "--dry-run"}, + "BACKSCROLL_MUTATION_WAIT=100ms") +if err := waitForChild(t, blocked, 10*time.Second); err == nil { + t.Fatal("busy remediation unexpectedly succeeded") +} +assertStderrContains(t, blocked, "sync_in_progress") +if strings.Contains(blocked.stdout.String(), "recovery dry run") { + t.Fatalf("blocked remediation emitted recovery output: %q", blocked.stdout.String()) +} + +if err := os.WriteFile(release, []byte("release"), 0o600); err != nil { t.Fatal(err) } +requireChildSuccess(t, owner, 10*time.Second) + +retry := startCoordinationChild(t, []string{"recover", "--from", dbPath, "--dry-run"}) +requireChildSuccess(t, retry, 10*time.Second) +if !strings.Contains(retry.stdout.String(), "recovery dry run") { + t.Fatalf("retry stdout=%q", retry.stdout.String()) +} +assertCounterLines(t, counter, 1) +``` + +The final counter assertion proves remediation did not invoke pre-handler sync. Do not assert absence of the persistent sidecar; ADR 0003 requires it to remain. + +- [ ] **Step 2: Run subprocess coordination repeatedly** + +```bash +go test ./cmd/backscroll -run '^(TestStartupCoordination.*Remediation|TestRecoverDryRunBypassesPreparation)' -count=5 +``` + +Expected: PASS without timing flakes. + +- [ ] **Step 3: Commit cross-process remediation evidence** + +```bash +git add cmd/backscroll/startup_coordination_process_test.go +git commit -m "test(recovery): prove remediation contention" +``` + +### Task 5: Update living guidance and run all gates + +**Files:** +- Modify: `CLAUDE.md` — startup coordination and core pipeline sections. +- Modify: `docs/sync.md` — owner/follower/remediation behavior. + +**Interfaces:** +- Consumes: completed remediation startup behavior. +- Produces: maintainer/user operational contract and final verification evidence. + +- [ ] **Step 1: Update command classes in living documentation** + +Document: + +```text +snapshot-read: search, list, patterns, status, validate +metadata-read: config +mutation: annotate, purge, rebuild +remediation: recover +``` + +State that remediation acquires/retains the mutation-grade lock but skips ordinary compatible-open and pre-handler sync; apply runs post-install sync under the same lease. + +Remove claims that `recover` first performs mandatory startup sync or proceeds by carrying a prior startup failure into its handler. + +- [ ] **Step 2: Run focused tests** + +```bash +go test ./cmd/backscroll -run 'TestCoordinateStartup|TestEveryOperationalCommand|TestRemediation|TestRecover|TestStartupCoordination.*Remediation' -count=1 +``` + +Expected: PASS. + +- [ ] **Step 3: Run race and package tests** + +```bash +go test -race ./cmd/backscroll ./internal/recovery ./internal/startuplock +go test ./... +``` + +Expected: PASS. + +- [ ] **Step 4: Verify Windows lock compilation** + +```bash +GOOS=windows GOARCH=amd64 go test -c ./internal/startuplock -o /tmp/startuplock-windows.test.exe +``` + +Expected: compile exits 0. + +- [ ] **Step 5: Run repository gates** + +```bash +just check +just test +just ci +``` + +Expected: PASS; aggregate coverage at least 85 percent. + +- [ ] **Step 6: Commit documentation** + +```bash +git add CLAUDE.md docs/sync.md +git commit -m "docs(recovery): document remediation startup" +``` + +- [ ] **Step 7: Record review evidence** + +PR description must report: + +- exact ALTER-built dry-run test output; +- prepare and pre-handler sync call counts of zero for remediation; +- immediate and busy-owner lease behavior; +- subprocess repetition count; +- race, Windows compile, and CI results; +- confirmation that no new flag, command, lock, or persistent state was added. diff --git a/docs/superpowers/plans/2026-08-24-semantic-lineage-closure.md b/docs/superpowers/plans/2026-08-24-semantic-lineage-closure.md new file mode 100644 index 0000000..3497216 --- /dev/null +++ b/docs/superpowers/plans/2026-08-24-semantic-lineage-closure.md @@ -0,0 +1,770 @@ +# Semantic Lineage Closure Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Replace textual/checksum lineage identity with conservative semantic schema identity and prove every catalog fixture from V1 onward reaches current head without skipped migration paths. + +**Architecture:** Keep `internal/compat` stateless and read-only. Derive a semantic signature from SQLite structural metadata plus token-canonicalized DDL, keep migration rows as separate provenance, and key catalog lookup by `(AppliedVersion, Signature)`. Drive one lossless production-path migration test from every distinct physical fixture in the catalog. + +**Tech Stack:** Go 1.26.2, `database/sql`, `modernc.org/sqlite`, SHA-256, embedded JSON/SQL fixtures, stdlib testing, Just + +**Spec:** `docs/superpowers/specs/2026-08-24-semantic-lineage-closure-design.md` + +## Global Constraints + +- Unknown semantic shapes remain fail-closed with `unsupported_lineage`. +- Do not add V14 signature exceptions, compatibility flags, fallback catalogs, repair modes, or persistent state. +- Preserve exact contents of string literals and quoted identifiers. +- Preserve behaviorally meaningful DDL distinctions: constraints, expressions, triggers, partial indexes, foreign keys, generated columns, and virtual-table arguments. +- Migration checksum history is provenance: it is inspected and preserved but does not multiply equivalent semantic identities. +- Every distinct physical fixture remains a closure input even when signatures converge. +- The closure test contains no fixture allowlist and no conditional `t.Skip` path. +- Do not modify migration V1–V14 SQL bodies. +- All tests are hermetic and use `t.TempDir()` or in-memory SQLite. +- Follow RED → GREEN → REFACTOR, focused tests before package and repository gates. + +--- + +## File map + +| Path | Responsibility | +|---|---| +| `internal/compat/sql_canonical.go` | Token-canonicalize SQLite DDL while preserving literals and quoted identifiers. | +| `internal/compat/sql_canonical_test.go` | Equivalent and behaviorally distinct SQL evidence. | +| `internal/compat/schema.go` | Inspect structural shape; load migration provenance separately; produce semantic signature. | +| `internal/compat/schema_test.go` | Semantic signature, provenance separation, malformed-ledger, and unsupported-shape tests. | +| `internal/compat/types.go` | Keep `SchemaShape` stable; no new public compatibility state. | +| `internal/compat/catalog.go` | Composite shape lookup and collision-safe lineage attachment. | +| `internal/compat/catalog_test.go` | Composite-key, collision, fixture retention, and checked-in signature tests. | +| `internal/compat/regenerate_manifest.go` | Deterministically regenerate semantic signatures while preserving every fixture mapping. | +| `internal/compat/testdata/release-schemas/manifest.json` | Regenerated semantic signatures for all physical fixtures. | +| `internal/storage/recovery_records.go` | Validate recovery inputs by complete `SchemaShape`, not signature alone. | +| `internal/storage/recovery_records_test.go` | Recovery lookup rejects wrong version even if semantic signature matches. | +| `internal/storage/migration_plan_test.go` | Catalog-derived V1→head closure matrix with losslessness assertions and no skips. | +| `CLAUDE.md` | Record semantic identity and mandatory historical closure invariant. | + +### Task 1: Introduce conservative DDL token canonicalization + +**Files:** +- Create: `internal/compat/sql_canonical.go` +- Create: `internal/compat/sql_canonical_test.go` + +**Interfaces:** +- Consumes: SQLite DDL strings already loaded through `loadSQLiteObjects`. +- Produces: `func canonicalSQL(string) string`; `normalizeSQL` is removed after all internal callers migrate. + +- [ ] **Step 1: Write equivalence tests that fail on current normalization** + +Create `internal/compat/sql_canonical_test.go` with table-driven tests: + +```go +package compat + +import "testing" + +func TestCanonicalSQLEquivalentRepresentationsMatch(t *testing.T) { + tests := []struct { + name string + left, right string + }{ + { + name: "punctuation whitespace", + left: "CREATE TABLE s ( a TEXT, b INTEGER )", + right: "create table s(a text,b integer)", + }, + { + name: "comments are not semantics", + left: "CREATE TABLE s (a TEXT /* historical layout */, b INTEGER)", + right: "CREATE TABLE s(a TEXT,b INTEGER)", + }, + { + name: "alter appended layout", + left: "CREATE TABLE s (a TEXT, b INTEGER\n)", + right: "CREATE TABLE s (a TEXT, b INTEGER)", + }, + { + name: "comment token boundary", + left: "CREATE TABLE s (a/**/TEXT)", + right: "CREATE TABLE s (a TEXT)", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got, want := canonicalSQL(tt.left), canonicalSQL(tt.right); got != want { + t.Fatalf("canonical SQL differs\nleft: %q\nright: %q", got, want) + } + }) + } +} +``` + +- [ ] **Step 2: Write non-equivalence tests before implementation** + +Add exact distinctions required by the spec: + +```go +func TestCanonicalSQLBehavioralDifferencesRemainDistinct(t *testing.T) { + tests := []struct { + name string + left, right string + }{ + {"literal", "CREATE TABLE s(a TEXT DEFAULT 'x y')", "CREATE TABLE s(a TEXT DEFAULT 'x y')"}, + {"quoted identifier", `CREATE TABLE "a b"(id INTEGER)`, `CREATE TABLE "a b"(id INTEGER)`}, + {"check", "CREATE TABLE s(a INTEGER CHECK(a > 0))", "CREATE TABLE s(a INTEGER CHECK(a >= 0))"}, + {"conflict", "CREATE TABLE s(a TEXT UNIQUE)", "CREATE TABLE s(a TEXT UNIQUE ON CONFLICT REPLACE)"}, + {"deferrable", "CREATE TABLE s(a INTEGER REFERENCES p(id))", "CREATE TABLE s(a INTEGER REFERENCES p(id) DEFERRABLE)"}, + {"partial index", "CREATE INDEX i ON s(a) WHERE a > 0", "CREATE INDEX i ON s(a) WHERE a >= 0"}, + {"trigger", "CREATE TRIGGER t AFTER INSERT ON s BEGIN SELECT 1; END", "CREATE TRIGGER t AFTER INSERT ON s BEGIN SELECT 2; END"}, + {"fts tokenizer", "CREATE VIRTUAL TABLE f USING fts5(body, tokenize='porter')", "CREATE VIRTUAL TABLE f USING fts5(body, tokenize='trigram')"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if canonicalSQL(tt.left) == canonicalSQL(tt.right) { + t.Fatalf("%s unexpectedly collided", tt.name) + } + }) + } +} +``` + +- [ ] **Step 3: Run RED and verify the missing function is the reason** + +Run: + +```bash +go test ./internal/compat -run '^TestCanonicalSQL' +``` + +Expected: FAIL to compile with `undefined: canonicalSQL`. + +- [ ] **Step 4: Implement token canonicalization in a focused file** + +Create `internal/compat/sql_canonical.go` with these exact private interfaces: + +```go +package compat + +import ( + "strconv" + "strings" + "unicode" +) + +type sqlTokenKind byte + +const ( + sqlBare sqlTokenKind = iota + sqlLiteral + sqlQuotedIdentifier + sqlPunctuation +) + +type sqlToken struct { + kind sqlTokenKind + text string +} + +func canonicalSQL(input string) string { + tokens := scanSQLTokens(input) + var encoded strings.Builder + for _, token := range tokens { + encoded.WriteByte(byte('0' + token.kind)) + encoded.WriteByte(':') + encoded.WriteString(strconv.Itoa(len(token.text))) + encoded.WriteByte(':') + encoded.WriteString(token.text) + } + return encoded.String() +} +``` + +Implement `scanSQLTokens(input string) []sqlToken` as a single forward scanner with these rules, in this order: + +1. Skip Unicode/ASCII whitespace. +2. On `--`, consume through CR/LF and emit no token. +3. On `/*`, consume through the first `*/` and emit no token. +4. On `'`, copy through the closing quote, preserving `''` escapes, and emit `sqlLiteral` with exact bytes. +5. On `"`, `` ` ``, or `[`, copy through the matching delimiter, preserving doubled `""`, and emit `sqlQuotedIdentifier` with exact bytes. +6. Match the longest punctuation/operator from `->>`, `||`, `->`, `<<`, `>>`, `<=`, `>=`, `==`, `!=`, `<>`, then single-byte `(),;.+-*/%<>=&|~` and emit `sqlPunctuation`. +7. Consume a bare token until whitespace, quote, comment opener, or punctuation; emit `sqlBare` with `strings.ToLower`. +8. If a quote or block comment is unterminated, preserve the remaining bytes in its token. `sqlite_master` should not contain malformed DDL, but deterministic output is required. + +The length-prefixed token encoding is mandatory. Do not join raw tokens with a sentinel byte: SQL literals can contain arbitrary control bytes and would make delimiter-based serialization ambiguous. + +Use one helper per quoted form: + +```go +func scanSingleQuoted(input string, start int) (string, int) +func scanDelimitedIdentifier(input string, start int, open, close byte, doubledClose bool) (string, int) +func longestSQLOperator(input string, offset int) (string, bool) +func isSQLPunctuation(ch byte) bool +``` + +`scanSQLTokens` must always advance at least one byte. Use `unicode.IsSpace` only when decoding a valid rune; SQL punctuation and quotes remain byte-oriented so exact literal bytes survive. + +- [ ] **Step 5: Run canonicalization tests and confirm GREEN** + +Run: + +```bash +go test ./internal/compat -run '^TestCanonicalSQL' +``` + +Expected: PASS. + +- [ ] **Step 6: Keep the new canonicalizer isolated until identity changes atomically** + +Do not change `schema.go` in this task. The existing `normalizeSQL` remains the production path until Task 2 can switch canonicalization, provenance, catalog keys, and manifest signatures together without leaving the repository red. + +- [ ] **Step 7: Run the complete canonicalizer test file** + +Run: + +```bash +go test ./internal/compat -run '^TestCanonicalSQL' +``` + +Expected: PASS. Existing compat and storage behavior remains unchanged because `canonicalSQL` is not wired yet. + +- [ ] **Step 8: Commit the independently reviewed canonicalizer** + +```bash +git add internal/compat/sql_canonical.go internal/compat/sql_canonical_test.go +git commit -m "feat(compat): canonicalize semantic ddl tokens" +``` + +### Task 2: Separate migration provenance and key catalog lookup by full shape + +**Files:** +- Modify: `internal/compat/schema.go:22-135,184-235,247-253,415-632` +- Modify: `internal/compat/schema_test.go:1-655` +- Modify: `internal/compat/catalog.go:17-199` +- Modify: `internal/compat/catalog_test.go:70-150,379-435` +- Modify: `internal/compat/regenerate_manifest.go:17-139` +- Modify: `internal/compat/testdata/release-schemas/manifest.json` +- Modify: `internal/storage/recovery_records.go:19-52` +- Modify: `internal/storage/recovery_records_test.go` + +**Interfaces:** +- Consumes: `SchemaShape{AppliedVersion, Signature}` and `canonicalSQL` from Task 1. +- Produces: + +```go +type lineageKey struct { + appliedVersion int + signature string +} + +func (c Catalog) ByShape(shape SchemaShape) (Lineage, bool) +func (c Catalog) IsKnownShape(shape SchemaShape) bool +func (c Catalog) CurrentShape() SchemaShape +``` + +`BySignature`, `IsKnownSignature`, and `CurrentSignature` are removed after all callers migrate. + +- [ ] **Step 1: Write failing provenance-separation tests** + +Add to `schema_test.go`: + +```go +func TestSemanticSignatureIgnoresMigrationChecksumHistory(t *testing.T) { + const schema = ` + CREATE TABLE schema_migrations ( + version INTEGER PRIMARY KEY, + name TEXT NOT NULL, + applied_on TEXT NOT NULL, + checksum TEXT NOT NULL + ); + CREATE TABLE items (id INTEGER PRIMARY KEY, body TEXT NOT NULL); + ` + left := openSchema(t, schema+`INSERT INTO schema_migrations VALUES (1, 'v1', 'clock-a', 'published');`) + defer left.Close() + right := openSchema(t, schema+`INSERT INTO schema_migrations VALUES (1, 'v1', 'clock-b', 'development');`) + defer right.Close() + + leftShape, err := inspectShape(context.Background(), left) + if err != nil { t.Fatal(err) } + rightShape, err := inspectShape(context.Background(), right) + if err != nil { t.Fatal(err) } + if leftShape.AppliedVersion != rightShape.AppliedVersion || leftShape.Signature != rightShape.Signature { + t.Fatalf("equivalent current shapes differ: left=%+v right=%+v", leftShape.SchemaShape, rightShape.SchemaShape) + } +} +``` + +Add a structural-name test because PRAGMA returns preserved spelling even for case-insensitive unquoted identifiers: + +```go +func TestSemanticSignatureNormalizesUnquotedStructuralNames(t *testing.T) { + left := openSchema(t, `CREATE TABLE Items (Body TEXT); CREATE INDEX ItemIndex ON Items(Body);`) + defer left.Close() + right := openSchema(t, `create table items (body text); create index itemindex on items(body);`) + defer right.Close() + leftShape, err := inspectShape(context.Background(), left) + if err != nil { t.Fatal(err) } + rightShape, err := inspectShape(context.Background(), right) + if err != nil { t.Fatal(err) } + if leftShape.Signature != rightShape.Signature { + t.Fatalf("unquoted case changed semantic signature: left=%s right=%s", leftShape.Signature, rightShape.Signature) + } +} +``` + +Add a second test proving version remains outside the hash but inside identity: + +```go +func TestCatalogIdentityIncludesAppliedVersion(t *testing.T) { + catalog := Catalog{lineages: map[lineageKey]Lineage{ + {appliedVersion: 1, signature: "sha256:same"}: {shape: SchemaShape{AppliedVersion: 1, Signature: "sha256:same"}}, + {appliedVersion: 2, signature: "sha256:same"}: {shape: SchemaShape{AppliedVersion: 2, Signature: "sha256:same"}}, + }} + if _, ok := catalog.ByShape(SchemaShape{AppliedVersion: 1, Signature: "sha256:same"}); !ok { t.Fatal("v1 missing") } + if _, ok := catalog.ByShape(SchemaShape{AppliedVersion: 2, Signature: "sha256:same"}); !ok { t.Fatal("v2 missing") } + if _, ok := catalog.ByShape(SchemaShape{AppliedVersion: 3, Signature: "sha256:same"}); ok { t.Fatal("unknown v3 accepted") } +} +``` + +- [ ] **Step 2: Run RED** + +Run: + +```bash +go test ./internal/compat -run '^(TestSemanticSignatureIgnoresMigrationChecksumHistory|TestCatalogIdentityIncludesAppliedVersion)$' +``` + +Expected: checksum test FAILS because migration rows affect the hash; catalog test FAILS to compile because `lineageKey` and `ByShape` do not exist. + +- [ ] **Step 3: Wire canonical DDL and load migration provenance outside semantic records** + +Replace every `normalizeSQL` call in `schema.go` with `canonicalSQL`, delete the old `normalizeSQL` implementation, and move any still-relevant quoted-literal/identifier tests to `sql_canonical_test.go`. + +Add: + +```go +func canonicalStructuralName(name string) string { return strings.ToLower(name) } +``` + +Use it only in records that contribute to the signature: `sqliteObject.table/name`, column record names, index record table/name, and PRAGMA-returned index column names. Lowercase trimmed column type affinity in `tableColumn.signature`. Keep `columnsByTable` keys at their actual SQLite spelling for migration planning. Full canonical DDL remains in the signature, so quoted identifier contents still remain distinct. + +Replace `loadSchemaMigrationRecords` with: + +```go +type migrationProvenance struct { + version int + name string + checksum string +} + +func loadMigrationProvenance(ctx context.Context, q Queryer) (rows []migrationProvenance, appliedVersion int, err error) +``` + +Use the existing ordered SQL query and scan logic. Validate during iteration: + +- version must be positive; +- versions must increase strictly; +- name and checksum must be non-empty; +- `appliedVersion` is the final version. + +Return wrapped errors naming `schema_migrations`. Do not append migration rows to the `records` slice hashed by `inspectShape`. + +In `inspectShape`, preserve support for an empty database with no ledger: + +```go +var provenance []migrationProvenance +appliedVersion := 0 +if hasObject(objects, "table", "schema_migrations") { + provenance, appliedVersion, err = loadMigrationProvenance(ctx, q) + if err != nil { return inspectedShape{}, err } +} +``` + +Extend the private shape only: + +```go +type inspectedShape struct { + SchemaShape + columnsByTable map[string]map[string]bool + migrationProvenance []migrationProvenance +} +``` + +Set `migrationProvenance: provenance` in the result. Do not expose provenance in `SchemaShape` or migration plans. + +- [ ] **Step 4: Implement composite catalog identity** + +Change `Catalog.lineages` to `map[lineageKey]Lineage`; add `currentShape SchemaShape`; implement: + +```go +func keyForShape(shape SchemaShape) lineageKey { + return lineageKey{appliedVersion: shape.AppliedVersion, signature: shape.Signature} +} + +func (c Catalog) ByShape(shape SchemaShape) (Lineage, bool) { + lineage, ok := c.lineages[keyForShape(shape)] + return lineage, ok +} + +func (c Catalog) IsKnownShape(shape SchemaShape) bool { + _, ok := c.ByShape(shape) + return ok +} + +func (c Catalog) CurrentShape() SchemaShape { return c.currentShape } +``` + +Update `attachLineages` to key each fixture by `keyForShape(shape)`. Set `currentShape` from the latest release's `AppliedVersion` and `Signature`. + +Update `InspectIndex`: + +```go +lineage, ok := defaultCatalog.ByShape(shape.SchemaShape) +``` + +Update catalog tests to assert `CurrentShape()` rather than `CurrentSignature()`. + +- [ ] **Step 5: Make recovery validate complete shape identity** + +Rename the private helper: + +```go +func readRecordsForShape(ctx context.Context, q compat.Queryer, shape compat.SchemaShape) ([]models.IndexedRecord, *compat.Diagnostic, error) +``` + +Call `catalog.IsKnownShape(shape)` and include both version and signature in unsupported summaries. Update `ReadRecoveryInputFromQueryer` to pass `plan.From`. + +Add to `recovery_records_test.go`: + +```go +func TestReadRecordsForShapeRejectsKnownSignatureWithUnknownVersion(t *testing.T) { + dbPath := createFixtureDatabase(t, "v14.sql") + db, err := OpenReadOnly(dbPath) + if err != nil { t.Fatal(err) } + defer db.Close() + catalog, err := compat.LoadCatalog() + if err != nil { t.Fatal(err) } + shape := catalog.CurrentShape() + shape.AppliedVersion++ + records, diag, err := readRecordsForShape(context.Background(), db.DB(), shape) + if err != nil { t.Fatal(err) } + if records != nil || diag == nil || diag.Code != compat.CodeUnsupportedLineage { + t.Fatalf("records=%v diagnostic=%+v", records, diag) + } +} +``` + +- [ ] **Step 6: Regenerate semantic signatures exactly once** + +Run from repository root: + +```bash +REGEN_MANIFEST=1 go test ./internal/compat -run '^TestRegenerateManifestOnNormalizationChange$' -v +``` + +Expected: PASS and `manifest.json` updated. Fixture provenance hashes remain unchanged because fixture bytes were not edited. Multiple physical fixtures may now share a semantic signature. + +- [ ] **Step 7: Run the affected packages to GREEN** + +```bash +go test ./internal/compat ./internal/storage +``` + +Expected: PASS. No test observes stale checked-in signatures or calls the removed signature-only catalog API. + +- [ ] **Step 8: Commit the atomic semantic identity switch** + +```bash +git add internal/compat/schema.go internal/compat/schema_test.go internal/compat/catalog.go \ + internal/compat/catalog_test.go internal/compat/regenerate_manifest.go \ + internal/compat/testdata/release-schemas/manifest.json \ + internal/storage/recovery_records.go internal/storage/recovery_records_test.go +git commit -m "fix(compat): identify semantic schema lineages" +``` + +### Task 3: Harden semantic collision and fixture-retention guarantees + +**Files:** +- Modify: `internal/compat/catalog.go:139-199` +- Modify: `internal/compat/catalog_test.go:120-150,297-435` +- Modify: `internal/compat/regenerate_manifest.go:17-139` +- Modify: `internal/compat/testdata/release-schemas/manifest.json` + +**Interfaces:** +- Consumes: composite identity and semantic signatures from Tasks 1–2. +- Produces: deterministic checked-in semantic signatures and collision-safe `attachLineages`. + +- [ ] **Step 1: Write collision-consistency tests before tightening attachment** + +Replace reflection-based collision testing with tests through `attachLineages`. Add a helper fixture catalog and these cases: + +```go +func TestAttachLineagesAcceptsEquivalentPhysicalHistories(t *testing.T) { + catalog := Catalog{UnmanifestedFixtures: []catalogFixture{ + {Fixture: "fresh.sql", Signature: "sha256:same", AppliedVersion: 13, HasSourceMetadata: false, Provenance: "fresh"}, + {Fixture: "alter.sql", Signature: "sha256:same", AppliedVersion: 13, HasSourceMetadata: false, Provenance: "alter"}, + }, LatestGoRelease: "v3.2.5", Releases: []catalogRelease{{Tag: "v3.2.5", Fixture: "fresh.sql", Signature: "sha256:same", AppliedVersion: 13}}} + if err := catalog.attachLineages(); err != nil { t.Fatal(err) } +} + +func TestAttachLineagesRejectsAmbiguousSemanticCollision(t *testing.T) { + catalog := Catalog{UnmanifestedFixtures: []catalogFixture{ + {Fixture: "with.sql", Signature: "sha256:same", AppliedVersion: 5, HasSourceMetadata: true, Provenance: "with"}, + {Fixture: "without.sql", Signature: "sha256:same", AppliedVersion: 5, HasSourceMetadata: false, Provenance: "without"}, + }} + if err := catalog.attachLineages(); err == nil || !strings.Contains(err.Error(), "ambiguous semantic collision") { + t.Fatalf("collision error = %v", err) + } +} +``` + +- [ ] **Step 2: Run RED** + +Run: + +```bash +go test ./internal/compat -run '^TestAttachLineages' +``` + +Expected: ambiguous collision test FAILS because the later map entry silently overwrites the first. + +- [ ] **Step 3: Reject differing plans for one composite key** + +In `attachLineages`, before assignment: + +```go +key := keyForShape(shape) +lineage := Lineage{shape: shape, remainingSteps: remainingStepsFor(fixture.AppliedVersion, fixture.HasSourceMetadata)} +if existing, ok := lineages[key]; ok { + if !reflect.DeepEqual(existing.remainingSteps, lineage.remainingSteps) { + return fmt.Errorf("ambiguous semantic collision version=%d signature=%s: fixtures disagree on remaining migration plan", shape.AppliedVersion, shape.Signature) + } + continue +} +lineages[key] = lineage +``` + +Prefer a small `sameMigrationSteps(left, right []MigrationStep) bool` helper instead of adding `reflect` to production code. Comparing remaining plans captures `HasSourceMetadata` differences that alter V6 planning. + +- [ ] **Step 4: Add a regenerator retention test** + +Add a temp-directory test using two physical fixture files that differ only in DDL formatting and migration checksum: + +```go +func TestRegenerateManifestRetainsConvergedPhysicalFixtures(t *testing.T) { + dir := t.TempDir() + manifestPath := filepath.Join(dir, "manifest.json") + base := `CREATE TABLE schema_migrations (version INTEGER PRIMARY KEY, name TEXT NOT NULL, applied_on TEXT NOT NULL, checksum TEXT NOT NULL);` + fresh := base + `INSERT INTO schema_migrations VALUES (1,'v1','clock','published'); CREATE TABLE items (id INTEGER, body TEXT);` + altered := base + `INSERT INTO schema_migrations VALUES (1,'v1','clock','development'); CREATE TABLE items(id INTEGER,body TEXT);` + if err := os.WriteFile(filepath.Join(dir, "fresh.sql"), []byte(fresh), 0o644); err != nil { t.Fatal(err) } + if err := os.WriteFile(filepath.Join(dir, "altered.sql"), []byte(altered), 0o644); err != nil { t.Fatal(err) } + manifest := `{ + "FirstGoRelease":"v0.3.7","LatestGoRelease":"v3.2.5", + "Releases":[ + {"Tag":"v0.3.7","Fixture":"fresh.sql","ProvenanceSHA256":"old","Signature":"sha256:old-a","AppliedVersion":1}, + {"Tag":"v3.2.5","Fixture":"altered.sql","ProvenanceSHA256":"old","Signature":"sha256:old-b","AppliedVersion":1} + ] + }` + if err := os.WriteFile(manifestPath, []byte(manifest), 0o644); err != nil { t.Fatal(err) } + if err := RegenerateManifestJSON(manifestPath); err != nil { t.Fatal(err) } + regenerated, err := loadCatalogFromPath(manifestPath) + if err != nil { t.Fatal(err) } + if len(regenerated.Releases) != 2 { t.Fatalf("release mappings=%d want 2", len(regenerated.Releases)) } + if regenerated.Releases[0].Fixture == regenerated.Releases[1].Fixture { + t.Fatalf("physical histories collapsed: %+v", regenerated.Releases) + } + if regenerated.Releases[0].Signature != regenerated.Releases[1].Signature { + t.Fatalf("equivalent semantic signatures differ: %+v", regenerated.Releases) + } +} +``` + +The test changes no embedded fixture and proves regeneration updates mappings rather than deduplicating them. + +- [ ] **Step 5: Verify catalog signatures, collision groups, and fixture retention** + +Run: + +```bash +go test ./internal/compat -run '^(TestReleaseSchemaFixtureSignaturesMatchCheckedInSQL|TestAttachLineages|TestRegenerateManifest|TestCheckedInReleaseSchemaManifestIsComplete)$' -v +``` + +Expected: PASS. + +- [ ] **Step 6: Commit collision and regeneration hardening** + +```bash +git add internal/compat/catalog.go internal/compat/catalog_test.go internal/compat/regenerate_manifest.go +git commit -m "test(compat): guard semantic lineage collisions" +``` + +### Task 4: Enforce lossless migration closure from every physical fixture + +**Files:** +- Modify: `internal/storage/migration_plan_test.go:17-124,1480-1580` + +**Interfaces:** +- Consumes: `compat.LoadCatalog`, composite semantic identities, production `OpenCompatible`, and existing sentinel helpers. +- Produces: `TestEveryCatalogFixtureReachesCurrentSemanticHead` as the mandatory historical closure gate. + +- [ ] **Step 1: Replace the misleading closure test and delete all exclusions** + +Rename `TestCatalogGoLineagesUpgradeLosslessly` to: + +```go +func TestEveryCatalogFixtureReachesCurrentSemanticHead(t *testing.T) +``` + +Keep catalog-derived distinct fixture collection. Delete the V13 name check and both `t.Skip` lines completely. Do not add a replacement condition. + +At the start, assert corpus coverage: + +```go +required := []string{ + "v1.sql", "v2.sql", "v3.sql", "v3-no-source-metadata.sql", + "v4.sql", "v5-with-source-metadata.sql", "v5-without-source-metadata.sql", + "v6.sql", "v7.sql", "v8.sql", "v9.sql", "v10.sql", "v11.sql", "v12.sql", + "v13.sql", "v13-legacy-existing-schema-migrations.sql", + "v13-legacy-alter-built.sql", "v13-development-alter-built.sql", "v14.sql", +} +for _, name := range required { + if !seen[name] { t.Fatalf("catalog closure corpus lacks %s", name) } +} +``` + +This list is an assertion that known historical evidence remains inventoried, not an allowlist controlling execution. The loop still executes every catalog fixture, including future ones. + +- [ ] **Step 2: Strengthen final-shape and ledger assertions** + +After `OpenCompatible`, require: + +```go +plan, diag, err := compat.InspectIndex(context.Background(), db.DB()) +if err != nil || diag != nil { t.Fatalf("inspect current head error=%v diagnostic=%+v", err, diag) } +if len(plan.Steps) != 0 { t.Fatalf("fixture %s retained steps: %+v", fixture.name, plan.Steps) } +if plan.From != catalog.CurrentShape() { + t.Fatalf("fixture %s reached shape %+v, want %+v", fixture.name, plan.From, catalog.CurrentShape()) +} +``` + +Keep these losslessness checks inside every fixture subtest: + +```go +assertSearchItems(t, db.DB(), want.SearchItems) +assertSearchItemsByUUID(t, db.DB(), want.ToolSearchItems) +assertTableSentinels(t, db.DB(), want) +assertFTSQueryable(t, db.DB(), "sentinelterm", len(want.SearchItems)) +assertToolFTSQueryable(t, db.DB(), "sentinelcmd", len(want.ToolSearchItems)) +``` + +Retain the snapshot assertions already keyed by `fixture.expectSnapshot`. Keep the documented historical V9 checksum adjustment for `v13-development-alter-built.sql`; provenance is preserved even though it no longer affects semantic identity. + +- [ ] **Step 3: Run the exact issue regression** + +Run: + +```bash +go test ./internal/storage -run '^TestEveryCatalogFixtureReachesCurrentSemanticHead/(v13-legacy-alter-built.sql|v13-development-alter-built.sql)$' -v +``` + +Expected: both subtests PASS; no `SKIP`, and no `9cdad03b…` or `f6a081b9…` final verification failure. + +- [ ] **Step 4: Run the complete V1→head matrix** + +Run: + +```bash +go test ./internal/storage -run '^TestEveryCatalogFixtureReachesCurrentSemanticHead$' -v +``` + +Expected: every physical fixture PASS, zero skipped subtests. + +- [ ] **Step 5: Remove the obsolete single-path #52 regression** + +Delete `TestMigratedFixtureSignatureIsInCatalog`; the complete matrix strictly supersedes its V1-only assertion. Do not keep two differently named closure guarantees. + +- [ ] **Step 6: Commit the enforceable closure boundary** + +```bash +git add internal/storage/migration_plan_test.go +git commit -m "test(storage): require every lineage to reach head" +``` + +### Task 5: Document the invariant and verify the affected real lineage safely + +**Files:** +- Modify: `CLAUDE.md` — replace the whitespace-only #52 decision paragraph with semantic identity and closure requirements. + +**Interfaces:** +- Consumes: completed semantic identity and closure matrix. +- Produces: maintainer guidance and release evidence for #58. + +- [ ] **Step 1: Update living architecture guidance** + +Document these exact rules in `CLAUDE.md`: + +- schema identity is `(AppliedVersion, semantic Signature)`; +- migration rows are provenance, not signature input; +- canonical SQL discards comments/formatting but preserves literals, quoted identifiers, constraints, expressions, triggers, indexes, and virtual-table configuration; +- every physical catalog fixture is an upgrade target; +- no recognized fixture may be skipped; +- every new migration must pass `TestEveryCatalogFixtureReachesCurrentSemanticHead`. + +Keep the observed #52/#58 history concise; do not retain statements claiming full DDL text and migration rows remain part of identity. + +- [ ] **Step 2: Run focused and package verification** + +```bash +go test ./internal/compat ./internal/storage -run 'TestCanonicalSQL|TestSemanticSignature|TestAttachLineages|TestEveryCatalogFixtureReachesCurrentSemanticHead|TestReadRecoveryInput' -v +go test ./internal/compat ./internal/storage ./internal/recovery +``` + +Expected: PASS with no skipped closure fixtures. + +- [ ] **Step 3: Run repository gates** + +```bash +just check +just test +just ci +``` + +Expected: PASS; aggregate statement coverage at least 85 percent. + +- [ ] **Step 4: Build a dev binary and create an online backup of the affected database** + +```bash +go build -o /tmp/backscroll-issue58 ./cmd/backscroll +smoke_dir="$(mktemp -d)" +sqlite3 "$HOME/.backscroll.db" ".backup '$smoke_dir/affected.db'" +``` + +Expected: dev binary builds; backup command exits 0. Never point the development binary at the original database. + +- [ ] **Step 5: Verify the copied real `f6a081b9…` lineage reaches head** + +```bash +BACKSCROLL_DATABASE_PATH="$smoke_dir/affected.db" /tmp/backscroll-issue58 status +BACKSCROLL_DATABASE_PATH="$smoke_dir/affected.db" /tmp/backscroll-issue58 validate --json +``` + +Expected: both commands exit 0; neither output contains `unsupported_lineage`, `f6a081b9`, or `migration_failed`. The original `$HOME/.backscroll.db` remains untouched. + +If `sqlite3` is unavailable, report the smoke as pending instead of copying live SQLite files with `cp`; fixture closure remains the automated gate. + +- [ ] **Step 6: Commit guidance** + +```bash +git add CLAUDE.md +git commit -m "docs(compat): require semantic lineage closure" +``` + +- [ ] **Step 7: Record final evidence for review** + +Include in the PR description: + +- exact issue #58 subtests and complete fixture count; +- explicit zero-skip statement; +- regenerated signature collision count and validation result; +- focused/package/CI commands; +- copied-real-database smoke result or explicit pending reason; +- confirmation that no V14 signature-specific runtime branch was added. diff --git a/docs/superpowers/specs/2026-08-24-semantic-lineage-closure-design.md b/docs/superpowers/specs/2026-08-24-semantic-lineage-closure-design.md new file mode 100644 index 0000000..598ea6c --- /dev/null +++ b/docs/superpowers/specs/2026-08-24-semantic-lineage-closure-design.md @@ -0,0 +1,340 @@ +# Semantic Lineage Closure for Backscroll Issue #58 + +**Status:** Approved design + +**Date:** 2026-08-24 + +**Issue:** #58 + +## Decision summary + +Backscroll will replace textual migration-lineage identity with a hybrid semantic schema identity and make migration closure from every recognized historical fixture to the current schema a mandatory, no-skip test boundary. + +A schema's current meaning will select its migration plan. Historical `schema_migrations` rows remain inspectable provenance, but formatting differences and known historical checksum variants will not multiply otherwise equivalent schema identities. Unknown semantic shapes continue to fail closed. + +The `recover` command also becomes a true remediation path: it retains startup-lock ownership but bypasses ordinary compatible-open and startup-sync preparation, because recovery must remain reachable when normal preparation cannot open the index. + +## Context + +Issue #58 reproduces a compatibility defect previously observed in issues #41 and #52. Backscroll recognizes multiple V13 schemas created through historical migration paths. Migration V14 appends `file_size` and `file_mtime` with `ALTER TABLE`, preserving textual differences inherited from each V13 input. Final verification compares the result against a catalog containing only one fresh-built V14 signature. + +Two recognized V13 inputs therefore produce unknown V14 signatures: + +- `v13-legacy-alter-built.sql` produces `sha256:9cdad03b571fd70df9c1045eae6469864b0e2a66498d7ec9f6dc7ad0af2b2122`. +- `v13-development-alter-built.sql` produces `sha256:f6a081b9df13b30fdac598103d558325c2636e4cec6625f77d1294d26bd47f89`. + +`compat.VerifyCurrentShape` rejects both results before commit. The transaction rolls back to V13 and every operational command remains blocked. The local Backscroll database reproduces the `f6a081b9…` failure. + +The existing systemic compatibility design already requires every supported Go lineage to migrate losslessly to head with no skipped closure evidence. The implementation violated that contract by excluding the two ALTER-built fixtures in `TestCatalogGoLineagesUpgradeLosslessly`: + +```go +if fixture.name == "v13-development-alter-built.sql" || fixture.name == "v13-legacy-alter-built.sql" { + t.Skip("legacy v13 ALTER-built fixtures are v13 compatibility cases, not v14 upgrade targets") +} +``` + +Those fixtures are upgrade targets by definition: Backscroll recognizes them and returns a V14 migration plan. The suite encountered the release blocker and hid it. + +The failure is not limited to V13. Any recognized shape from V1 onward can preserve construction-path differences through later migrations. A V13-only correction would repeat the same incomplete boundary. + +The advertised `recover --from … --dry-run` continuation is also circular. Root startup attempts the same incompatible migration before recovery and returns the same `unsupported_lineage` diagnostic instead of completing useful remediation. + +## Goals + +- Make equivalent SQLite schema shapes share one compatibility identity despite irrelevant DDL formatting or construction history. +- Preserve distinctions that change SQLite behavior: constraints, expressions, quoted identifiers, literals, triggers, partial indexes, foreign keys, and virtual-table configuration. +- Make every distinct checked-in historical fixture from V1 onward migrate through the real production path to head without skips. +- Make every future migration automatically exercise every supported historical fixture. +- Preserve indexed rows, UUID identities, satellite tables, FTS queryability, migration ledgers, and required snapshots. +- Keep unknown semantic shapes fail-closed. +- Make `recover --dry-run` reachable when ordinary startup preparation fails, without mutating input data. +- Remove the current ALTER-built lockout without adding V14-specific runtime branches. + +## Non-goals + +- Accept arbitrary third-party or manually corrupted SQLite schemas. +- Implement a complete SQLite parser or prove every possible SQL equivalence. +- Rewrite historical `schema_migrations` rows in user databases. +- Add a compatibility flag, fallback catalog, repair mode, or persistent compatibility state. +- Change migration V1–V14 SQL or data behavior. +- Change recovery union, conflict, or atomic replacement semantics beyond reachability. +- Treat release tags as a substitute for checked-in fixture evidence. + +## System-reduction boundary + +The design removes two accidental compatibility dimensions: + +1. DDL formatting is no longer an independent lineage dimension. +2. Migration checksum history is no longer mixed into current semantic shape identity. + +It does not add an exception table for V14 outcomes. One semantic catalog and one catalog-derived closure matrix replace per-migration signature multiplication. + +The only startup-policy extension is the already-established conceptual remediation class for `recover`. It adds no command or persisted state; it separates ordinary preparation from the command intended to operate when ordinary preparation fails. + +## Architecture + +### Hybrid semantic identity + +`internal/compat` inspects three independent evidence categories: + +1. **Current structural shape** + - applied migration version; + - regular and virtual tables; + - columns, affinity, nullability, defaults, primary-key order, and generated/hidden state where exposed; + - indexes, uniqueness, indexed-column order, expressions, and partial-index state; + - foreign-key actions; + - triggers and views. + +2. **Canonical auxiliary SQL** + - constraints and expressions not fully exposed by PRAGMA; + - trigger bodies; + - partial-index predicates and expression indexes; + - virtual-table module arguments, including FTS tokenizer and content configuration. + +3. **Migration provenance** + - version, name, and checksum rows from `schema_migrations`; + - fixture and release provenance in the manifest. + +The first two categories produce the semantic signature used for compatibility lookup. Provenance remains available for diagnostics, inventory, and losslessness assertions, but does not create a distinct current identity when semantic shape and applied version agree. + +### Conservative SQL canonicalization + +Canonical auxiliary SQL reuses a focused lexer rather than introducing a parser framework. Outside literals and quoted identifiers it: + +- removes comments while retaining token separation where required; +- ignores whitespace and whitespace adjacency to punctuation; +- tokenizes punctuation and operators deterministically; +- normalizes unquoted SQLite keywords and identifiers according to case-insensitive SQLite rules; +- preserves token order. + +It preserves exact contents and escapes inside single-quoted literals and double-quoted, backtick, or bracket identifiers. + +It does not normalize numeric expressions, reorder expressions, rewrite constraints, or infer deeper equivalence. Conservative false negatives are acceptable; false acceptance of behaviorally distinct schemas is not. + +### Catalog model + +The checked-in manifest remains the hermetic inventory of releases and observed historical shapes. Each physical fixture retains: + +- fixture bytes and provenance hash; +- applied version; +- release or observed-shape provenance; +- semantic signature; +- migration-ledger provenance. + +Multiple physical fixtures may share a semantic signature. A collision is valid only when they share applied version, structural evidence, canonical auxiliary semantics, and remaining migration plan. + +Lookup becomes conceptually `(applied_version, semantic_signature) → Lineage`. Existing `SchemaShape.Signature` and catalog method names may remain to limit API churn, but documentation changes their meaning from full textual/ledger identity to semantic identity. + +Manifest regeneration recomputes semantic signatures for every fixture. It never deletes a fixture because two histories converge: every physical fixture remains an independent migration-closure input. + +### Migration closure matrix + +A table-driven test enumerates every distinct fixture path from the catalog, including release fixtures and explicitly observed unmanifested shapes. It has no handwritten fixture allowlist and no conditional skip path. + +For each fixture it: + +1. Copies the fixture into `t.TempDir()`. +2. Seeds version-appropriate sentinels in every perennial table available at that version. +3. Records expected historical ledger rows. +4. Invokes production `OpenCompatible`. +5. Applies all remaining real migrations to current head. +6. Verifies current semantic shape and an empty remaining plan. +7. Verifies search rows, UUIDs, tool rows, satellites, and available derived records. +8. Verifies message and tool FTS queryability. +9. Verifies historical ledger rows remain intact and new authoritative rows are appended. +10. Verifies snapshots exist only for applicable destructive migrations. + +The input corpus begins with every supported shape, not V13 alone: + +- V1 and V2; +- V3 with and without `source_metadata`; +- V4; +- V5 with and without `source_metadata`; +- V6 through V12; +- all fresh-built, release-built, development, and ALTER-built V13 forms; +- V14 and each future head fixture; +- every newly documented observed shape. + +Because inputs come from the catalog and remaining steps come from the current migration catalog, adding V15 automatically exercises every historical input through V15. No recognized fixture may be relabeled as a non-upgrade target. + +### Recovery startup boundary + +`recover` currently uses general mutation startup. That forces ordinary compatible-open and startup sync before its handler, creating a loop when migration itself fails. + +Root startup will classify `recover` as remediation while retaining the canonical startup lock and exclusive ownership. A remediation invocation: + +1. Validates CLI and configuration inputs. +2. Acquires the existing startup lock under the mutation timeout contract. +3. Skips ordinary `OpenCompatible` preparation and mandatory pre-handler sync. +4. Enters the recovery handler, whose adapters open active and `--from` inputs read-only. +5. In apply mode, retains the lock through replacement, verification, and post-install sync. +6. In `--dry-run`, performs no input-data writes. + +A continuation is valid only if it reaches the remediation planner rather than reproducing the same startup diagnostic. + +## Data flow + +```text +historical SQLite + ├── structural PRAGMA evidence ─────┐ + ├── canonical auxiliary DDL ────────┼── semantic shape + └── migration ledger ─ provenance ──┘ │ + ▼ + semantic catalog lookup + │ + ┌─────────────────────┴─────────────────────┐ + │ │ + supported semantic shape unknown shape + │ │ + real migration plan unsupported_lineage + │ + transaction + semantic verification + │ + recognized semantic head +``` + +Recovery uses a separate startup edge: + +```text +recover → startup lock → recovery planner → dry-run report + └→ verified apply → replacement → post-install sync +``` + +## Error handling + +### Unknown semantics + +A readable database whose semantic signature is absent remains `unsupported_lineage`. Equivalent formatting, comments, punctuation spacing, and known ledger variants must not trigger it. + +### Migration provenance + +Historical checksums are provenance, not current semantic identity. Unknown provenance does not override a fully recognized semantic shape. Contradictory, duplicated, non-monotonic, or structurally invalid ledger rows still fail because they make the applied-version claim unreliable. + +### Failed migration + +Execution and final semantic verification remain within one transaction. Any failure rolls back schema and data changes. Diagnostics identify the starting fixture/shape, failed step, and produced semantic signature. + +### Semantic collision + +If two fixtures share a signature but disagree on applied version, structural evidence, auxiliary semantics, or remaining steps, catalog loading fails. No arbitrary map winner is accepted. + +### Recovery loop + +A recovery continuation that re-enters ordinary preparation is a test failure. `recover --dry-run` must produce a recovery plan or recovery-specific diagnostic after inspecting its source; it cannot repeat startup `migration_failed` as the primary result. + +## Testing strategy + +### Equivalent shapes must collide + +- whitespace around `(`, `)`, and `,`; +- inline versus multiline declarations; +- comments and comment-contained quotes; +- fresh-created versus ALTER-appended equivalent columns; +- case differences in unquoted SQL tokens. + +### Behaviorally distinct shapes must not collide + +- different literals or quoted identifiers; +- `CHECK` expressions; +- `ON CONFLICT` behavior; +- foreign-key actions or deferrability; +- generated-column expressions; +- partial-index predicates; +- trigger bodies; +- FTS tokenizer or virtual-table arguments. + +### Catalog evidence + +- Every release maps to an existing fixture with verified bytes. +- Every unmanifested fixture has observed-shape provenance. +- Collision groups agree on version, evidence, and remaining plan. +- Regeneration is deterministic. +- Converging semantic identities never remove fixture histories. + +### Migration closure + +`TestEveryCatalogFixtureReachesCurrentSemanticHead` exercises every physical fixture without a skip branch. Failures report fixture, starting version/signature, failed migration, and produced signature. + +Focused #58 tests retain the exact ALTER-built signatures as evidence but add no runtime signature exceptions. + +### Recovery + +- `recover --dry-run` against the development ALTER-built fixture reaches planning. +- Dry-run preserves database bytes, WAL/SHM state, and row counts. +- Apply retains startup-lock ownership through post-install sync. +- Concurrent mutation behavior retains the existing five-second bound. + +### Repository gates + +```bash +go test ./internal/compat ./internal/storage ./internal/recovery ./cmd/backscroll +just check +just test +just ci +``` + +The closure matrix must report zero skipped fixture cases. Aggregate coverage remains at least 85 percent. + +## File boundaries + +| Path | Responsibility | +|---|---| +| `internal/compat/schema.go` | Hybrid semantic inspection and auxiliary-SQL canonicalization. | +| `internal/compat/schema_test.go` | Equivalence and non-equivalence evidence. | +| `internal/compat/catalog.go` | Semantic lookup and collision validation. | +| `internal/compat/catalog_test.go` | Manifest accountability and deterministic regeneration. | +| `internal/compat/regenerate_manifest.go` | Recompute signatures without deleting converged histories. | +| `internal/compat/testdata/release-schemas/manifest.json` | Fixture provenance and semantic signatures. | +| `internal/storage/migration_plan_test.go` | Catalog-derived closure matrix and losslessness assertions. | +| `cmd/backscroll/startup_commands.go` | Explicit remediation classification for `recover`. | +| `cmd/backscroll/startup_coordination.go` | Lock retention with remediation preparation bypass. | +| `cmd/backscroll/recover_test.go` and startup tests | Dry-run reachability, no diagnostic loop, and lock behavior. | +| `docs/adr/0004-identidad-semantica-y-cierre-de-linajes.md` | Versioned architectural decision. | + +No new production package is required. The lexer may move to one focused file inside `internal/compat` if needed for reviewability; no public parser abstraction is introduced. + +## Alternatives rejected + +### Add missing V14 signatures + +This unblocks current databases but preserves signature multiplication. V15 could produce one new head signature per historical formatting/checksum combination. + +### Normalize punctuation only + +This fixes #58's immediate textual difference but leaves migration history and other accidental DDL differences mixed into current identity. + +### Build a complete SQLite parser + +An AST could model deeper equivalence but introduces a major subsystem for SQLite dialect details, triggers, virtual tables, and future syntax. The hybrid model relies on SQLite metadata first. + +### Trust version and checksum alone + +Equal version labels have represented divergent shapes, while different checksum histories can reach equivalent shapes. This ignores current database semantics. + +### Keep recovery in ordinary mutation startup + +This preserves one startup branch but makes remediation depend on the condition it must remediate, producing a circular continuation. + +## Rollout and reversibility + +No user-database migration is required. Semantic signatures are computed at runtime and stored only in the checked-in catalog. + +Implementation, regenerated manifest, and tests can be reverted together without changing database bytes or historical ledger rows. Recovery startup classification is separately revertible, although reverting it restores the continuation loop. + +A development build must verify the real `~/.backscroll.db` affected by `f6a081b9…`. Release evidence includes successful `status`, a zero-skip closure matrix, and `recover --dry-run` reaching recovery planning. + +## Acceptance criteria + +- [ ] Every distinct catalog fixture from V1 onward reaches current head through `OpenCompatible`. +- [ ] The closure matrix contains no fixture exclusions or conditional skips. +- [ ] Both #58 ALTER-built V13 fixtures migrate to the recognized semantic head. +- [ ] Equivalent fresh-built and ALTER-built schemas share a semantic signature. +- [ ] Behaviorally distinct DDL remains distinct. +- [ ] Ledger provenance no longer multiplies equivalent identities. +- [ ] Invalid or contradictory ledgers remain fail-closed. +- [ ] Collision groups agree on version, evidence, and remaining plan. +- [ ] Rows, UUIDs, satellites, FTS, ledger rows, and required snapshots survive every path. +- [ ] A future migration automatically extends every historical fixture path. +- [ ] `recover --dry-run` bypasses ordinary compatible-open, reaches planning, and performs no input-data mutation. +- [ ] No runtime branch special-cases the V14 failure signatures. +- [ ] Focused packages, `just check`, `just test`, and `just ci` pass. diff --git a/docs/sync.md b/docs/sync.md index 9949475..b2f12d5 100644 --- a/docs/sync.md +++ b/docs/sync.md @@ -3,22 +3,32 @@ estado: Completed --- # Sync and Indexing -Backscroll has no public `sync` command. Ingestion is integrated into operational commands: active global input manifests are validated, changed inputs are detected by SHA-256, and only new or changed content is indexed. +Backscroll has no public `sync` command. Ingestion is integrated into ordinary operational startup: active global input manifests are validated, changed inputs are detected by SHA-256, and only new or changed content is indexed. -Every operational command validates active manifests and attempts one incremental -sync before executing. Session, plan, and Markdown files are ingestion inputs; -SQLite is the perennial record used by search, list, patterns, status, and validate. -Use `--source-path` on search as a filter, paired with query text, for database-backed retrieval scoped to a known input path. +Startup behavior is command-classed, not one-size-fits-all: + +```text +snapshot-read: search, list, patterns, status, validate +metadata-read: config +mutation: annotate, purge, rebuild +remediation: recover +``` + +Snapshot-read, metadata-read, and mutation owners validate active manifests and attempt one incremental sync before executing. Remediation (`recover`) is different: it acquires and retains the mutation-grade startup lock, skips ordinary compatible-open/index preparation and pre-handler sync, and lets the recovery handler inspect or replace an index that ordinary startup might reject. Session, plan, and Markdown files are ingestion inputs; SQLite is the perennial record used by search, list, patterns, status, and validate. Use `--source-path` on search as a filter, paired with query text, for database-backed retrieval scoped to a known input path. ## Coordinated startup pipeline ```text validated invocation + -> classify command -> try .startup-sync.lock - -> owner: prepare/migrate -> incremental sync -> command - -> busy snapshot read: compatible OpenReadOnly -> stderr warning -> query WAL snapshot - -> busy config: stderr warning -> print validated config - -> busy mutation: wait <=5s -> owner or retryable sync_in_progress + -> snapshot-read owner: prepare/migrate -> incremental sync -> release lock -> command + -> busy snapshot-read: compatible OpenReadOnly -> stderr warning -> query WAL snapshot + -> metadata-read owner: prepare/migrate -> incremental sync -> release lock -> command + -> busy metadata-read: stderr warning -> print validated config without opening the DB + -> mutation owner: prepare/migrate -> incremental sync -> retain lock -> command + -> remediation owner: retain mutation-grade lock -> skip ordinary compatible-open and pre-handler sync -> command + -> busy mutation/remediation: wait <=5s -> owner or retryable sync_in_progress ``` The lock sidecar is an empty file that persists with mode `0600`; it is never deleted, and only the OS advisory lock on that file represents ownership. Startup coordination is local-host only and assumes a trusted local filesystem. @@ -29,7 +39,7 @@ The lock sidecar is an empty file that persists with mode `0600`; it is never de # Show the active manifests and resolved paths. backscroll config -# Commands perform startup sync before querying SQLite. +# Snapshot-read, metadata-read, and mutation owners perform startup sync before the handler. backscroll search --text "migration plan" backscroll list --order timestamp:desc --limit 20 backscroll patterns --kind templates --min-support 5 @@ -37,7 +47,7 @@ backscroll status --json backscroll validate --json ``` -Human startup sync writes progress and warnings to stderr. JSON/robot startup progress is discarded so stdout remains machine-readable, and invalid active manifests fail during preflight instead of being silently ignored. Busy followers emit `sync_in_progress` warnings to stderr; read-safe followers use the last committed WAL snapshot, config followers print validated configuration without opening the database, and mutation followers wait up to five seconds before returning a retryable failure. +Human startup sync writes progress and warnings to stderr. JSON/robot startup progress is discarded so stdout remains machine-readable, and invalid active manifests fail during preflight instead of being silently ignored. Busy followers emit `sync_in_progress` warnings to stderr; read-safe followers use the last committed WAL snapshot, config followers print validated configuration without opening the database, and mutation/remediation followers wait up to five seconds before returning a retryable failure. `backscroll recover --dry-run` reports without post-install sync; `backscroll recover` apply runs post-install sync under the same retained remediation lease before printing its report. A search scoped to a known input path stays database-backed: @@ -52,7 +62,7 @@ backscroll search --text "permission denied" --source-path "*/example/*.jsonl" - backscroll rebuild ``` -`rebuild` is non-destructive. The mandatory root startup sync runs first and prepares the database. The rebuild handler does not perform a second sync: it re-derives both FTS5 indexes from the perennial `search_items` table, backfills derived templates/corrections/tool events from stored text where possible, and re-resolves project identities. It does not discard sessions whose files have expired. +`rebuild` is non-destructive. Mutation-class startup sync runs first and prepares the database. The rebuild handler does not perform a second sync: it re-derives both FTS5 indexes from the perennial `search_items` table, backfills derived templates/corrections/tool events from stored text where possible, and re-resolves project identities. It does not discard sessions whose files have expired. Use `rebuild` after index-recovery work or when derived search structures need regeneration. It is not a substitute for a removed manual sync command. `backscroll purge --before ` is the explicit deletion path. @@ -98,7 +108,7 @@ Plans and external Markdown documents are also declared as inputs. Use `decode.f Backscroll stores a SHA-256 hash for each indexed input. Unchanged files are skipped on later startup syncs. Files with stable message UUIDs are updated append-only; legacy or UUID-less inputs retain wipe-and-reload behavior while the source exists. -Startup coordination uses owner/follower branches: an owner acquires the canonical lock and performs prepare/migrate/sync before the handler runs; a read-safe follower validates the existing database read-only and continues on a compatible snapshot; a mutation follower waits up to five seconds for ownership or fails retryably with `sync_in_progress`. WAL snapshot followers remain compatible only on the same local host. +Startup coordination uses owner/follower branches: snapshot-read, metadata-read, and mutation owners acquire the canonical lock and perform prepare/migrate/sync before the handler runs; a remediation owner acquires and retains the same mutation-grade lock but bypasses ordinary compatible-open and pre-handler sync; a read-safe follower validates the existing database read-only and continues on a compatible snapshot; metadata-read followers avoid opening the database; mutation and remediation followers wait up to five seconds for ownership or fail retryably with `sync_in_progress`. WAL snapshot followers remain compatible only on the same local host. The SQLite database is the perennial event store, not a disposable cache. When a source file expires, its indexed rows remain available. Only `purge` removes retained data explicitly. diff --git a/internal/compat/catalog.go b/internal/compat/catalog.go index 7995852..f581780 100644 --- a/internal/compat/catalog.go +++ b/internal/compat/catalog.go @@ -20,8 +20,8 @@ type Catalog struct { Releases []catalogRelease UnmanifestedFixtures []catalogFixture - lineages map[string]Lineage - currentSignature string + lineages map[lineageKey]Lineage + currentShape SchemaShape } type catalogRelease struct { @@ -47,22 +47,26 @@ type Lineage struct { remainingSteps []MigrationStep } -func (c Catalog) BySignature(signature string) (Lineage, bool) { - lineage, ok := c.lineages[signature] +type lineageKey struct { + appliedVersion int + signature string +} + +func keyForShape(shape SchemaShape) lineageKey { + return lineageKey{appliedVersion: shape.AppliedVersion, signature: shape.Signature} +} + +func (c Catalog) ByShape(shape SchemaShape) (Lineage, bool) { + lineage, ok := c.lineages[keyForShape(shape)] return lineage, ok } -// IsKnownSignature returns true if the signature is in the lineage catalog. -// Use this to check if a schema is recognized, independent of its migration -// status or other semantic properties. -func (c Catalog) IsKnownSignature(signature string) bool { - _, ok := c.lineages[signature] +func (c Catalog) IsKnownShape(shape SchemaShape) bool { + _, ok := c.ByShape(shape) return ok } -func (c Catalog) CurrentSignature() string { - return c.currentSignature -} +func (c Catalog) CurrentShape() SchemaShape { return c.currentShape } func (l Lineage) RemainingSteps() []MigrationStep { steps := make([]MigrationStep, len(l.remainingSteps)) @@ -177,27 +181,63 @@ func (c Catalog) schemaFixtures() []catalogFixture { } func (c *Catalog) attachLineages() error { - lineages := map[string]Lineage{} + lineages := map[lineageKey]Lineage{} for _, fixture := range c.schemaFixtures() { shape := SchemaShape{AppliedVersion: fixture.AppliedVersion, Signature: fixture.Signature} - lineages[fixture.Signature] = Lineage{ + lineage := Lineage{ shape: shape, remainingSteps: remainingStepsFor(fixture.AppliedVersion, fixture.HasSourceMetadata), } + key := keyForShape(shape) + if existing, ok := lineages[key]; ok { + if !sameMigrationSteps(existing.remainingSteps, lineage.remainingSteps) { + return fmt.Errorf("ambiguous semantic collision version=%d signature=%s: fixtures disagree on remaining migration plan", shape.AppliedVersion, shape.Signature) + } + continue + } + lineages[key] = lineage } - for _, release := range c.Releases { - if release.Tag == c.LatestGoRelease { - c.currentSignature = release.Signature - break + maxVersion := 0 + for _, lineage := range lineages { + if lineage.shape.AppliedVersion > maxVersion { + maxVersion = lineage.shape.AppliedVersion } } - if c.currentSignature == "" { - return fmt.Errorf("release schema catalog latest release %q has no signature", c.LatestGoRelease) + if maxVersion == 0 { + return fmt.Errorf("release schema catalog has no fixtures") + } + + var maxShape SchemaShape + for _, lineage := range lineages { + shape := lineage.shape + if shape.AppliedVersion != maxVersion { + continue + } + if maxShape.Signature == "" { + maxShape = shape + continue + } + if shape.Signature != maxShape.Signature { + return fmt.Errorf("ambiguous current head version=%d: signatures %s and %s", maxVersion, maxShape.Signature, shape.Signature) + } } + c.currentShape = maxShape c.lineages = lineages return nil } +func sameMigrationSteps(left, right []MigrationStep) bool { + if len(left) != len(right) { + return false + } + for i := range left { + if left[i] != right[i] { + return false + } + } + return true +} + func compareSemver(left, right string) int { var lMajor, lMinor, lPatch int var rMajor, rMinor, rPatch int diff --git a/internal/compat/catalog_test.go b/internal/compat/catalog_test.go index 38028b6..2cbd9b2 100644 --- a/internal/compat/catalog_test.go +++ b/internal/compat/catalog_test.go @@ -85,23 +85,25 @@ func TestLoadCatalogUsesCheckedInSignaturesWithoutExecutingFixtureSQL(t *testing if err != nil { t.Fatalf("load catalog executed fixture SQL or rejected checked-in signature data: %v", err) } - if got := catalog.CurrentSignature(); got != "sha256:poisoned" { - t.Fatalf("current signature = %q, want checked-in signature", got) + if got := catalog.CurrentShape(); got != (SchemaShape{AppliedVersion: 13, Signature: "sha256:poisoned"}) { + t.Fatalf("current shape = %+v, want checked-in shape", got) } } -func TestCurrentSignatureFollowsLatestReleaseMapping(t *testing.T) { +func TestCurrentShapeFollowsMaxAppliedVersionNotLatestRelease(t *testing.T) { fixtureSQL := []byte("CREATE TABLE schema_migrations (version INTEGER PRIMARY KEY, name TEXT NOT NULL, applied_on TEXT NOT NULL, checksum TEXT NOT NULL);") fixtureSHA := fmt.Sprintf("%x", sha256.Sum256(fixtureSQL)) withReleaseSchemaFS(t, fstest.MapFS{ "testdata/release-schemas/manifest.json": {Data: []byte(fmt.Sprintf(`{ "FirstGoRelease": "v0.3.7", - "LatestGoRelease": "v3.2.6", + "LatestGoRelease": "v3.2.5", "Releases": [ - {"Tag": "v0.3.7", "Fixture": "v13.sql", "ProvenanceSHA256": %q, "Signature": "sha256:old-latest", "AppliedVersion": 13}, - {"Tag": "v3.2.5", "Fixture": "v13.sql", "ProvenanceSHA256": %q, "Signature": "sha256:old-latest", "AppliedVersion": 13}, - {"Tag": "v3.2.6", "Fixture": "v14.sql", "ProvenanceSHA256": %q, "Signature": "sha256:new-latest", "AppliedVersion": 14} + {"Tag": "v0.3.7", "Fixture": "v13.sql", "ProvenanceSHA256": %q, "Signature": "sha256:latest-release", "AppliedVersion": 13}, + {"Tag": "v3.2.5", "Fixture": "v13.sql", "ProvenanceSHA256": %q, "Signature": "sha256:latest-release", "AppliedVersion": 13} + ], + "UnmanifestedFixtures": [ + {"Fixture": "v14.sql", "ProvenanceSHA256": %q, "Signature": "sha256:unmanifested-head", "AppliedVersion": 14, "Provenance": "local current fixture"} ] }`, fixtureSHA, fixtureSHA, fixtureSHA))}, "testdata/release-schemas/v13.sql": {Data: fixtureSQL}, @@ -112,8 +114,8 @@ func TestCurrentSignatureFollowsLatestReleaseMapping(t *testing.T) { if err != nil { t.Fatal(err) } - if got := catalog.CurrentSignature(); got != "sha256:new-latest" { - t.Fatalf("current signature = %q, want latest release mapping signature", got) + if got := catalog.CurrentShape(); got != (SchemaShape{AppliedVersion: 14, Signature: "sha256:unmanifested-head"}) { + t.Fatalf("current shape = %+v, want max applied version shape", got) } } @@ -375,67 +377,113 @@ func loadFixtureMigrationRows(t *testing.T, fixtureSQL []byte) []migrationRow { return result } -// TestCollisionConsistencyGuardsAgainstSignatureAmbiguity verifies that when -// whitespace normalization collapses multiple fixtures into the same signature, -// all colliding entries agree on AppliedVersion and HasSourceMetadata. If any -// collision group disagrees, the Catalog.BySignature map's winner is arbitrary -// and recovery could plan the wrong migration steps — a real bug. -func TestCollisionConsistencyGuardsAgainstSignatureAmbiguity(t *testing.T) { - catalog, err := LoadCatalog() - if err != nil { +func TestAttachLineagesAcceptsEquivalentPhysicalHistories(t *testing.T) { + catalog := Catalog{ + UnmanifestedFixtures: []catalogFixture{ + {Fixture: "fresh.sql", Signature: "sha256:same", AppliedVersion: 13, HasSourceMetadata: false, Provenance: "fresh"}, + {Fixture: "alter.sql", Signature: "sha256:same", AppliedVersion: 13, HasSourceMetadata: false, Provenance: "alter"}, + }, + LatestGoRelease: "v3.2.5", + Releases: []catalogRelease{ + {Tag: "v3.2.5", Fixture: "fresh.sql", Signature: "sha256:same", AppliedVersion: 13}, + }, + } + if err := catalog.attachLineages(); err != nil { t.Fatal(err) } +} - // Build collision groups: signature -> list of (AppliedVersion, HasSourceMetadata) - collisions := make(map[string][]struct { - source string - version int - hasMetaData bool - }) - - for _, release := range catalog.Releases { - collisions[release.Signature] = append(collisions[release.Signature], struct { - source string - version int - hasMetaData bool - }{fmt.Sprintf("release %s", release.Tag), release.AppliedVersion, release.HasSourceMetadata}) +func TestAttachLineagesAcceptsMultipleSignaturesBelowCurrentHead(t *testing.T) { + for i := 0; i < 200; i++ { + catalog := Catalog{ + Releases: []catalogRelease{ + {Tag: "v0.3.7", Fixture: "v3.sql", Signature: "sha256:v3-with-source", AppliedVersion: 3, HasSourceMetadata: true}, + {Tag: "v3.2.5", Fixture: "v13.sql", Signature: "sha256:v13", AppliedVersion: 13, HasSourceMetadata: false}, + }, + UnmanifestedFixtures: []catalogFixture{ + {Fixture: "v3-no-source-metadata.sql", Signature: "sha256:v3-without-source", AppliedVersion: 3, HasSourceMetadata: false, Provenance: "historical V3 without source_metadata"}, + {Fixture: "v5-with-source-metadata.sql", Signature: "sha256:v5-with-source", AppliedVersion: 5, HasSourceMetadata: true, Provenance: "historical V5 with source_metadata"}, + {Fixture: "v5-without-source-metadata.sql", Signature: "sha256:v5-without-source", AppliedVersion: 5, HasSourceMetadata: false, Provenance: "historical V5 without source_metadata"}, + {Fixture: "v14.sql", Signature: "sha256:v14", AppliedVersion: 14, HasSourceMetadata: false, Provenance: "semantic head"}, + }, + } + if err := catalog.attachLineages(); err != nil { + t.Fatalf("iteration %d: attach lineages rejected non-head signatures: %v", i, err) + } + if got := catalog.CurrentShape(); got != (SchemaShape{AppliedVersion: 14, Signature: "sha256:v14"}) { + t.Fatalf("iteration %d: current shape = %+v, want V14 head", i, got) + } } +} - for _, fixture := range catalog.UnmanifestedFixtures { - collisions[fixture.Signature] = append(collisions[fixture.Signature], struct { - source string - version int - hasMetaData bool - }{fmt.Sprintf("unmanifested %s", fixture.Fixture), fixture.AppliedVersion, fixture.HasSourceMetadata}) +func TestAttachLineagesRejectsAmbiguousCurrentHead(t *testing.T) { + catalog := Catalog{ + Releases: []catalogRelease{ + {Tag: "v3.2.5", Fixture: "fresh.sql", Signature: "sha256:head-a", AppliedVersion: 14}, + }, + UnmanifestedFixtures: []catalogFixture{ + {Fixture: "alter.sql", Signature: "sha256:head-b", AppliedVersion: 14, Provenance: "competing head"}, + }, + } + if err := catalog.attachLineages(); err == nil || !strings.Contains(err.Error(), "ambiguous current head version=14") { + t.Fatalf("current head ambiguity error = %v", err) } +} - // Check each collision group for consistency - for sig, entries := range collisions { - if len(entries) <= 1 { - // No collision; skip - continue - } +func TestAttachLineagesRejectsAmbiguousSemanticCollision(t *testing.T) { + catalog := Catalog{ + UnmanifestedFixtures: []catalogFixture{ + {Fixture: "with.sql", Signature: "sha256:same", AppliedVersion: 5, HasSourceMetadata: true, Provenance: "with"}, + {Fixture: "without.sql", Signature: "sha256:same", AppliedVersion: 5, HasSourceMetadata: false, Provenance: "without"}, + }, + } + if err := catalog.attachLineages(); err == nil || !strings.Contains(err.Error(), "ambiguous semantic collision") { + t.Fatalf("collision error = %v", err) + } +} - // All entries in this collision must agree on AppliedVersion and HasSourceMetadata - first := entries[0] - for i, entry := range entries[1:] { - if entry.version != first.version { - t.Errorf("signature %s has inconsistent AppliedVersion: %s says %d, %s says %d", - sig, first.source, first.version, entry.source, entry.version) - } - if entry.hasMetaData != first.hasMetaData { - t.Errorf("signature %s has inconsistent HasSourceMetadata: %s says %v, %s says %v", - sig, first.source, first.hasMetaData, entry.source, entry.hasMetaData) - } - if i == 0 { - t.Logf("collision group %s: %s, %s agree", sig[:16], first.source, entry.source) - } - } +func TestRegenerateManifestRetainsConvergedPhysicalFixtures(t *testing.T) { + dir := t.TempDir() + manifestPath := filepath.Join(dir, "manifest.json") + base := `CREATE TABLE schema_migrations (version INTEGER PRIMARY KEY, name TEXT NOT NULL, applied_on TEXT NOT NULL, checksum TEXT NOT NULL);` + fresh := base + `INSERT INTO schema_migrations VALUES (1,'v1','clock','published'); CREATE TABLE items (id INTEGER, body TEXT);` + altered := base + `INSERT INTO schema_migrations VALUES (1,'v1','clock','development'); CREATE TABLE items(id INTEGER,body TEXT);` + if err := os.WriteFile(filepath.Join(dir, "fresh.sql"), []byte(fresh), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "altered.sql"), []byte(altered), 0o644); err != nil { + t.Fatal(err) + } + manifest := `{ + "FirstGoRelease":"v0.3.7","LatestGoRelease":"v3.2.5", + "Releases":[ + {"Tag":"v0.3.7","Fixture":"fresh.sql","ProvenanceSHA256":"old","Signature":"sha256:old-a","AppliedVersion":1}, + {"Tag":"v3.2.5","Fixture":"altered.sql","ProvenanceSHA256":"old","Signature":"sha256:old-b","AppliedVersion":1} + ] + }` + if err := os.WriteFile(manifestPath, []byte(manifest), 0o644); err != nil { + t.Fatal(err) + } + if err := RegenerateManifestJSON(manifestPath); err != nil { + t.Fatal(err) + } + regenerated, err := loadCatalogFromPath(manifestPath) + if err != nil { + t.Fatal(err) + } + if len(regenerated.Releases) != 2 { + t.Fatalf("release mappings=%d want 2", len(regenerated.Releases)) + } + if regenerated.Releases[0].Fixture == regenerated.Releases[1].Fixture { + t.Fatalf("physical histories collapsed: %+v", regenerated.Releases) + } + if regenerated.Releases[0].Signature != regenerated.Releases[1].Signature { + t.Fatalf("equivalent semantic signatures differ: %+v", regenerated.Releases) } } // TestRegenerateManifestOnNormalizationChange is an optional helper test that -// regenerates manifest.json after normalizeSQL changes. Run with: +// regenerates manifest.json after canonical SQL changes. Run with: // go test -run TestRegenerateManifestOnNormalizationChange ./internal/compat -v func TestRegenerateManifestOnNormalizationChange(t *testing.T) { if os.Getenv("REGEN_MANIFEST") == "" { diff --git a/internal/compat/regenerate_manifest.go b/internal/compat/regenerate_manifest.go index 347c9ab..0d3ded5 100644 --- a/internal/compat/regenerate_manifest.go +++ b/internal/compat/regenerate_manifest.go @@ -11,9 +11,9 @@ import ( "strings" ) -// RegenerateManifestJSON reads all fixtures and regenerates the manifest.json with -// new signatures computed using the current normalizeSQL implementation. -// This is called after changing normalizeSQL to ensure all signatures remain valid. +// RegenerateManifestJSON reads all fixtures and regenerates manifest.json with +// signatures computed using the current schema inspection canonicalization. +// This is called after changing canonicalization to ensure all signatures remain valid. func RegenerateManifestJSON(manifestPath string) error { // Load the old manifest to preserve the release mappings and unmapped fixtures oldCatalog, err := loadCatalogFromPath(manifestPath) diff --git a/internal/compat/schema.go b/internal/compat/schema.go index e027536..0a88eb6 100644 --- a/internal/compat/schema.go +++ b/internal/compat/schema.go @@ -28,11 +28,11 @@ func InspectIndex(ctx context.Context, q Queryer) (MigrationPlan, *Diagnostic, e if defaultCatalogErr != nil { return plan, nil, fmt.Errorf("load schema catalog: %w", defaultCatalogErr) } - lineage, ok := defaultCatalog.BySignature(shape.Signature) + lineage, ok := defaultCatalog.ByShape(shape.SchemaShape) if !ok { return plan, &Diagnostic{ Code: CodeUnsupportedLineage, - Summary: fmt.Sprintf("unsupported index schema %s", shape.Signature), + Summary: fmt.Sprintf("unsupported index schema version %d signature %s", shape.AppliedVersion, shape.Signature), }, nil } plan.Steps = lineage.RemainingSteps() @@ -55,7 +55,8 @@ func VerifyCurrentShape(ctx context.Context, q Queryer) error { type inspectedShape struct { SchemaShape - columnsByTable map[string]map[string]bool + columnsByTable map[string]map[string]bool + migrationProvenance []migrationProvenance } func inspectShape(ctx context.Context, q Queryer) (inspectedShape, error) { @@ -66,15 +67,14 @@ func inspectShape(ctx context.Context, q Queryer) (inspectedShape, error) { columnsByTable := map[string]map[string]bool{} var records []string + var provenance []migrationProvenance appliedVersion := 0 if hasObject(objects, "table", "schema_migrations") { - migrationRows, maxVersion, err := loadSchemaMigrationRecords(ctx, q) + provenance, appliedVersion, err = loadMigrationProvenance(ctx, q) if err != nil { return inspectedShape{}, err } - records = append(records, migrationRows...) - appliedVersion = maxVersion } virtualTables := map[string]bool{} @@ -102,7 +102,7 @@ func inspectShape(ctx context.Context, q Queryer) (inspectedShape, error) { } continue } - records = append(records, schemaRecord(object.typ, object.table, object.name, "", normalizeSQL(object.sql))) + records = append(records, schemaRecord(object.typ, canonicalStructuralName(object.table), canonicalStructuralName(object.name), "", canonicalSQL(object.sql))) if object.typ != "table" { continue } @@ -111,7 +111,7 @@ func inspectShape(ctx context.Context, q Queryer) (inspectedShape, error) { return inspectedShape{}, err } for _, column := range columns { - records = append(records, schemaRecord("column", object.name, column.name, column.signature(), "")) + records = append(records, schemaRecord("column", canonicalStructuralName(object.name), canonicalStructuralName(column.name), column.signature(), "")) if columnsByTable[object.name] == nil { columnsByTable[object.name] = map[string]bool{} } @@ -131,7 +131,8 @@ func inspectShape(ctx context.Context, q Queryer) (inspectedShape, error) { AppliedVersion: appliedVersion, Signature: fmt.Sprintf("sha256:%x", signatureBytes), }, - columnsByTable: columnsByTable, + columnsByTable: columnsByTable, + migrationProvenance: provenance, }, nil } @@ -192,9 +193,9 @@ func loadRegularTableRecords(ctx context.Context, q Queryer, object sqliteObject // collision because PRAGMA-derived fields omit DDL semantics such as // AUTOINCREMENT, COLLATE, ON CONFLICT, and DEFERRABLE. The only non-canonical // altered shape we support is an explicit checked-in fixture/signature. - records := []string{schemaRecord("table", object.table, object.name, "", normalizeSQL(object.sql))} + records := []string{schemaRecord("table", canonicalStructuralName(object.table), canonicalStructuralName(object.name), "", canonicalSQL(object.sql))} for _, column := range columns { - records = append(records, schemaRecord("column", object.name, column.name, column.signature(), "")) + records = append(records, schemaRecord("column", canonicalStructuralName(object.name), canonicalStructuralName(column.name), column.signature(), "")) } indexRecords, err := loadIndexRecords(ctx, q, object.name) if err != nil { @@ -204,8 +205,14 @@ func loadRegularTableRecords(ctx context.Context, q Queryer, object sqliteObject return records, columns, nil } -func loadSchemaMigrationRecords(ctx context.Context, q Queryer) (records []string, maxVersion int, err error) { - rows, err := q.QueryContext(ctx, ` +type migrationProvenance struct { + version int + name string + checksum string +} + +func loadMigrationProvenance(ctx context.Context, q Queryer) (rows []migrationProvenance, appliedVersion int, err error) { + sqlRows, err := q.QueryContext(ctx, ` SELECT version, name, checksum FROM schema_migrations ORDER BY version @@ -213,25 +220,40 @@ func loadSchemaMigrationRecords(ctx context.Context, q Queryer) (records []strin if err != nil { return nil, 0, fmt.Errorf("query schema_migrations: %w", err) } - defer joinRowsCloseError(rows, &err) + defer joinRowsCloseError(sqlRows, &err) - for rows.Next() { - var version int - var name, checksum string - if scanErr := rows.Scan(&version, &name, &checksum); scanErr != nil { + previousVersion := 0 + for sqlRows.Next() { + var row migrationProvenance + if scanErr := sqlRows.Scan(&row.version, &row.name, &row.checksum); scanErr != nil { err = fmt.Errorf("scan schema_migrations: %w", scanErr) return nil, 0, err } - if version > maxVersion { - maxVersion = version + if row.version <= 0 { + err = fmt.Errorf("schema_migrations version must be positive: %d", row.version) + return nil, 0, err + } + if row.version <= previousVersion { + err = fmt.Errorf("schema_migrations versions must increase strictly: %d after %d", row.version, previousVersion) + return nil, 0, err + } + if row.name == "" { + err = fmt.Errorf("schema_migrations version %d has empty name", row.version) + return nil, 0, err } - records = append(records, schemaRecord("migration", "schema_migrations", fmt.Sprintf("%013d", version), name+"|"+checksum, "")) + if row.checksum == "" { + err = fmt.Errorf("schema_migrations version %d has empty checksum", row.version) + return nil, 0, err + } + previousVersion = row.version + appliedVersion = row.version + rows = append(rows, row) } - if rowsErr := rows.Err(); rowsErr != nil { + if rowsErr := sqlRows.Err(); rowsErr != nil { err = fmt.Errorf("read schema_migrations: %w", rowsErr) return nil, 0, err } - return records, maxVersion, nil + return rows, appliedVersion, nil } type tableColumn struct { @@ -247,9 +269,9 @@ type tableColumn struct { func (c tableColumn) signature() string { defaultValue := "" if c.defaultTo.Valid { - defaultValue = normalizeSQL(c.defaultTo.String) + defaultValue = canonicalSQL(c.defaultTo.String) } - return fmt.Sprintf("%013d:%s:%d:%s:%d:%d", c.cid, c.typ, c.notNull, defaultValue, c.pk, c.hidden) + return fmt.Sprintf("%013d:%s:%d:%s:%d:%d", c.cid, strings.ToLower(strings.TrimSpace(c.typ)), c.notNull, defaultValue, c.pk, c.hidden) } func loadTableColumns(ctx context.Context, q Queryer, table string) (columns []tableColumn, err error) { @@ -294,7 +316,7 @@ func loadIndexRecords(ctx context.Context, q Queryer, table string) ([]string, e return nil, err } metadata := fmt.Sprintf("unique=%d origin=%s partial=%d columns=%s", index.unique, index.origin, index.partial, strings.Join(columns, ",")) - records = append(records, schemaRecord("index", table, index.name, metadata, "")) + records = append(records, schemaRecord("index", canonicalStructuralName(table), canonicalStructuralName(index.name), metadata, "")) } return records, nil } @@ -338,7 +360,7 @@ func loadIndexColumns(ctx context.Context, q Queryer, index string) (columns []s } columnName := "" if name.Valid { - columnName = name.String + columnName = canonicalStructuralName(name.String) } columns = append(columns, fmt.Sprintf("%013d:%013d:%s", seqno, cid, columnName)) } @@ -350,6 +372,8 @@ func loadIndexColumns(ctx context.Context, q Queryer, index string) (columns []s return columns, nil } +func canonicalStructuralName(name string) string { return strings.ToLower(name) } + func remainingStepsFor(appliedVersion int, hasSourceMetadata bool) []MigrationStep { var steps []MigrationStep for _, step := range allMigrationSteps { @@ -412,226 +436,6 @@ func isFTSShadowObject(name string, virtualTables map[string]bool) bool { return false } -func normalizeSQL(sqlText string) string { - // Collapse runs of whitespace outside SQL lexical constructs (comments, string literals, identifiers), - // preserving whitespace within: - // - Single-quoted string literals (including '' escapes) - // - Double-quoted identifiers (including "" escapes) - // - Backtick-quoted identifiers - // - Bracket-quoted identifiers [...] - // - Line comments (-- until end-of-line) - // - Block comments (/* ... */ non-nesting) - // - // Defect fix #1: Apostrophes and quotes within comments do not affect lexer state. - // Defect fix #2: Double-quoted identifiers preserve inner whitespace (e.g., "my table" != "my table"). - - var result strings.Builder - lastWasSpace := false - - for i := 0; i < len(sqlText); i++ { - ch := sqlText[i] - - // Handle line comments (-- until end-of-line) - if ch == '-' && i+1 < len(sqlText) && sqlText[i+1] == '-' { - // Write the comment delimiter - result.WriteByte(ch) - i++ - result.WriteByte(sqlText[i]) - lastWasSpace = false - i++ - // Collect comment content until EOL, collapsing internal whitespace - var commentBuf strings.Builder - commentLastWasSpace := false - for i < len(sqlText) && sqlText[i] != '\n' { - if sqlText[i] == '\r' { - // Skip CR; will be normalized by outer whitespace logic - i++ - continue - } - if sqlText[i] == ' ' || sqlText[i] == '\t' { - if !commentLastWasSpace { - commentBuf.WriteByte(' ') - commentLastWasSpace = true - } - i++ - } else { - commentBuf.WriteByte(sqlText[i]) - commentLastWasSpace = false - i++ - } - } - // Write comment content, trimmed of trailing whitespace - commentContent := strings.TrimRight(commentBuf.String(), " \t") - result.WriteString(commentContent) - // If we found a newline, write it as a space (for collapsing purposes) - if i < len(sqlText) && sqlText[i] == '\n' { - result.WriteByte(' ') - lastWasSpace = true - i++ - } - i-- // Adjust for the outer loop's i++ - continue - } - - // Handle block comments (/* ... */ non-nesting) - if ch == '/' && i+1 < len(sqlText) && sqlText[i+1] == '*' { - // Write the comment opener - result.WriteByte(ch) - i++ - result.WriteByte(sqlText[i]) // '*' - lastWasSpace = false - i++ - // Collect comment content until */, collapsing internal whitespace - var commentBuf strings.Builder - commentLastWasSpace := false - for i < len(sqlText) { - if sqlText[i] == '*' && i+1 < len(sqlText) && sqlText[i+1] == '/' { - // End of comment found - break - } - if sqlText[i] == ' ' || sqlText[i] == '\t' || sqlText[i] == '\n' || sqlText[i] == '\r' { - if !commentLastWasSpace { - commentBuf.WriteByte(' ') - commentLastWasSpace = true - } - i++ - } else { - commentBuf.WriteByte(sqlText[i]) - commentLastWasSpace = false - i++ - } - } - // Write comment content, preserving structure (trim only leading/trailing multiples) - commentContent := strings.TrimSpace(commentBuf.String()) - if commentContent != "" { - result.WriteByte(' ') - result.WriteString(commentContent) - result.WriteByte(' ') - } else { - // Empty comment - result.WriteByte(' ') - } - // Write comment closer if found - if i < len(sqlText) && sqlText[i] == '*' && i+1 < len(sqlText) && sqlText[i+1] == '/' { - result.WriteByte('*') - i++ - result.WriteByte('/') - i++ - lastWasSpace = false - } - i-- // Adjust for the outer loop's i++ - continue - } - - // Handle single-quoted string literals (preserve whitespace inside) - if ch == '\'' { - result.WriteByte(ch) - lastWasSpace = false - i++ - // Copy everything until closing single quote, handling '' escape - for i < len(sqlText) { - if sqlText[i] == '\'' { - result.WriteByte('\'') - // Check if this is an escape (followed by another single quote) - if i+1 < len(sqlText) && sqlText[i+1] == '\'' { - result.WriteByte('\'') - i += 2 - } else { - // End of string literal - i++ - lastWasSpace = false - break - } - } else { - result.WriteByte(sqlText[i]) - i++ - } - } - i-- // Adjust for the outer loop's i++ - continue - } - - // Handle double-quoted identifiers (preserve whitespace inside) - if ch == '"' { - result.WriteByte(ch) - lastWasSpace = false - i++ - // Copy everything until closing double quote, handling "" escape - for i < len(sqlText) { - if sqlText[i] == '"' { - result.WriteByte('"') - // Check if this is an escape (followed by another double quote) - if i+1 < len(sqlText) && sqlText[i+1] == '"' { - result.WriteByte('"') - i += 2 - } else { - // End of identifier - i++ - lastWasSpace = false - break - } - } else { - result.WriteByte(sqlText[i]) - i++ - } - } - i-- // Adjust for the outer loop's i++ - continue - } - - // Handle backtick-quoted identifiers (preserve whitespace inside) - if ch == '`' { - result.WriteByte(ch) - lastWasSpace = false - i++ - // Copy everything until closing backtick - for i < len(sqlText) && sqlText[i] != '`' { - result.WriteByte(sqlText[i]) - i++ - } - if i < len(sqlText) && sqlText[i] == '`' { - result.WriteByte('`') - i++ - lastWasSpace = false - } - i-- // Adjust for the outer loop's i++ - continue - } - - // Handle bracket-quoted identifiers [...] (preserve whitespace inside) - if ch == '[' { - result.WriteByte(ch) - lastWasSpace = false - i++ - // Copy everything until closing bracket - for i < len(sqlText) && sqlText[i] != ']' { - result.WriteByte(sqlText[i]) - i++ - } - if i < len(sqlText) && sqlText[i] == ']' { - result.WriteByte(']') - i++ - lastWasSpace = false - } - i-- // Adjust for the outer loop's i++ - continue - } - - // Outside all special contexts: collapse whitespace - if ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r' { - if !lastWasSpace { - result.WriteByte(' ') - lastWasSpace = true - } - } else { - result.WriteByte(ch) - lastWasSpace = false - } - } - - return strings.TrimSpace(result.String()) -} - func schemaRecord(kind, table, name, columns, sqlText string) string { return strings.Join([]string{kind, table, name, columns, sqlText}, "|") } diff --git a/internal/compat/schema_test.go b/internal/compat/schema_test.go index de8dcf9..4eacb76 100644 --- a/internal/compat/schema_test.go +++ b/internal/compat/schema_test.go @@ -156,6 +156,68 @@ func TestInspectIndexMalformedMigrationMetadataReturnsError(t *testing.T) { } } +func TestSemanticSignatureIgnoresMigrationChecksumHistory(t *testing.T) { + const schema = ` + CREATE TABLE schema_migrations ( + version INTEGER PRIMARY KEY, + name TEXT NOT NULL, + applied_on TEXT NOT NULL, + checksum TEXT NOT NULL + ); + CREATE TABLE items (id INTEGER PRIMARY KEY, body TEXT NOT NULL); + ` + left := openSchema(t, schema+`INSERT INTO schema_migrations VALUES (1, 'v1', 'clock-a', 'published');`) + defer left.Close() + right := openSchema(t, schema+`INSERT INTO schema_migrations VALUES (1, 'v1', 'clock-b', 'development');`) + defer right.Close() + + leftShape, err := inspectShape(context.Background(), left) + if err != nil { + t.Fatal(err) + } + rightShape, err := inspectShape(context.Background(), right) + if err != nil { + t.Fatal(err) + } + if leftShape.AppliedVersion != rightShape.AppliedVersion || leftShape.Signature != rightShape.Signature { + t.Fatalf("equivalent current shapes differ: left=%+v right=%+v", leftShape.SchemaShape, rightShape.SchemaShape) + } +} + +func TestSemanticSignatureNormalizesUnquotedStructuralNames(t *testing.T) { + left := openSchema(t, `CREATE TABLE Items (Body TEXT); CREATE INDEX ItemIndex ON Items(Body);`) + defer left.Close() + right := openSchema(t, `create table items (body text); create index itemindex on items(body);`) + defer right.Close() + leftShape, err := inspectShape(context.Background(), left) + if err != nil { + t.Fatal(err) + } + rightShape, err := inspectShape(context.Background(), right) + if err != nil { + t.Fatal(err) + } + if leftShape.Signature != rightShape.Signature { + t.Fatalf("unquoted case changed semantic signature: left=%s right=%s", leftShape.Signature, rightShape.Signature) + } +} + +func TestCatalogIdentityIncludesAppliedVersion(t *testing.T) { + catalog := Catalog{lineages: map[lineageKey]Lineage{ + {appliedVersion: 1, signature: "sha256:same"}: {shape: SchemaShape{AppliedVersion: 1, Signature: "sha256:same"}}, + {appliedVersion: 2, signature: "sha256:same"}: {shape: SchemaShape{AppliedVersion: 2, Signature: "sha256:same"}}, + }} + if _, ok := catalog.ByShape(SchemaShape{AppliedVersion: 1, Signature: "sha256:same"}); !ok { + t.Fatal("v1 missing") + } + if _, ok := catalog.ByShape(SchemaShape{AppliedVersion: 2, Signature: "sha256:same"}); !ok { + t.Fatal("v2 missing") + } + if _, ok := catalog.ByShape(SchemaShape{AppliedVersion: 3, Signature: "sha256:same"}); ok { + t.Fatal("unknown v3 accepted") + } +} + func TestRegularTableSignatureIsConservativeForUnsupportedDDL(t *testing.T) { for _, tt := range []struct { name string @@ -281,8 +343,8 @@ func TestInspectIndexRecognizesObservedDevelopmentV13Shape(t *testing.T) { if err != nil { t.Fatal(err) } - // Signature recomputed after normalizeSQL made whitespace-insensitive (issue #52) - const observedSignature = "sha256:4d04377754986f0da2f61f2ab73889168f596eb84e1f02df96e23072072e9375" + // Signature recomputed after canonical SQL and provenance separation changed semantic lineage. + const observedSignature = "sha256:bc02e80bddf32a68341c7bb5f28d6b1b95b3a6877436478743ae260541494b3e" if shape.Signature != observedSignature { t.Fatalf("development V13 signature = %s, want %s", shape.Signature, observedSignature) } @@ -388,266 +450,3 @@ func openSchema(t *testing.T, schemaSQL string) *sql.DB { } return db } - -func TestNormalizeSQLIsWhitespaceInsensitiveOutsideStringLiterals(t *testing.T) { - tests := []struct { - name string - sql1 string - sql2 string - }{ - { - name: "multiple spaces collapsed to single space", - sql1: "CREATE TABLE foo (id INTEGER)", - sql2: "CREATE TABLE foo (id INTEGER)", - }, - { - name: "newlines and tabs collapsed to single space", - sql1: "CREATE\n TABLE\tfoo\n(\nid\nINTEGER\n)", - sql2: "CREATE TABLE foo ( id INTEGER )", - }, - { - name: "alter-built schema: inline vs multiline columns normalize the same", - sql1: "CREATE TABLE search_items (\n content_type TEXT DEFAULT 'text',\n extraction_version INTEGER,\n was_interrupted INTEGER\n);", - sql2: "CREATE TABLE search_items ( content_type TEXT DEFAULT 'text', extraction_version INTEGER, was_interrupted INTEGER );", - }, - { - name: "preserve space inside single-quoted literal", - sql1: "CREATE TABLE foo (body TEXT DEFAULT 'alpha beta')", - sql2: "CREATE TABLE foo (body TEXT DEFAULT 'alpha beta')", - }, - { - name: "double-single-quote escape inside literal", - sql1: "CREATE TABLE foo (body TEXT DEFAULT 'O''Brien')", - sql2: "CREATE TABLE foo (body TEXT DEFAULT 'O''Brien')", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - norm1 := normalizeSQL(tt.sql1) - norm2 := normalizeSQL(tt.sql2) - if norm1 != norm2 { - t.Fatalf("normalization differs: %q != %q", norm1, norm2) - } - }) - } -} - -func TestNormalizeSQLPreservesWhitespaceInStringLiterals(t *testing.T) { - // This is critical: whitespace inside string literals must be preserved exactly - sql := "CREATE TABLE foo (body TEXT DEFAULT 'alpha beta', CHECK (body <> 'gamma delta'))" - norm := normalizeSQL(sql) - if !strings.Contains(norm, "'alpha beta'") { - t.Fatalf("whitespace in first literal not preserved: %q", norm) - } - if !strings.Contains(norm, "'gamma delta'") { - t.Fatalf("whitespace in second literal not preserved: %q", norm) - } -} - -func TestNormalizeSQLHandlesApostropheInLineComment(t *testing.T) { - // Defect 1: apostrophe in line comment should not flip parser into quote mode - tests := []struct { - name string - sql string - want string - }{ - { - name: "apostrophe in line comment", - sql: "CREATE TABLE t ( -- don't\n a INTEGER,\n b TEXT\n)", - want: "CREATE TABLE t ( -- don't a INTEGER, b TEXT )", - }, - { - name: "apostrophe in line comment with real literal", - sql: "CREATE TABLE t ( -- don't do this\n body TEXT DEFAULT 'x y'\n)", - want: "CREATE TABLE t ( -- don't do this body TEXT DEFAULT 'x y' )", - }, - { - name: "the row's id in comment", - sql: "CREATE TABLE t ( a INTEGER -- the row's id\n)", - want: "CREATE TABLE t ( a INTEGER -- the row's id )", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - norm := normalizeSQL(tt.sql) - if norm != tt.want { - t.Fatalf("normalizeSQL(%q) =\n %q\nwant\n %q", tt.sql, norm, tt.want) - } - }) - } -} - -func TestNormalizeSQLHandlesApostropheInBlockComment(t *testing.T) { - // Apostrophe in block comment should not flip parser into quote mode - tests := []struct { - name string - sql string - want string - }{ - { - name: "apostrophe in block comment", - sql: "CREATE TABLE t ( /* don't */ a INTEGER )", - want: "CREATE TABLE t ( /* don't */ a INTEGER )", - }, - { - name: "quote in block comment", - sql: "CREATE TABLE t ( /* use 'single quotes' inside */ a INTEGER )", - want: "CREATE TABLE t ( /* use 'single quotes' inside */ a INTEGER )", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - norm := normalizeSQL(tt.sql) - if norm != tt.want { - t.Fatalf("normalizeSQL(%q) =\n %q\nwant\n %q", tt.sql, norm, tt.want) - } - }) - } -} - -func TestNormalizeSQLPreservesWhitespaceInDoubleQuotedIdentifiers(t *testing.T) { - // Defect 2: double-quoted identifiers must preserve inner whitespace - tests := []struct { - name string - sql string - want string - }{ - { - name: "double-quoted identifier with multiple spaces", - sql: `CREATE TABLE "my table" ( a INTEGER )`, - want: `CREATE TABLE "my table" ( a INTEGER )`, - }, - { - name: "double-quoted column name with spaces", - sql: `CREATE TABLE t ( "col name" INTEGER )`, - want: `CREATE TABLE t ( "col name" INTEGER )`, - }, - { - name: "double-quote escape in identifier", - sql: `CREATE TABLE "my""table" ( a INTEGER )`, - want: `CREATE TABLE "my""table" ( a INTEGER )`, - }, - { - name: "both single and double quoted identifiers", - sql: `CREATE TABLE "my table" ( id INTEGER, body TEXT DEFAULT 'x y' )`, - want: `CREATE TABLE "my table" ( id INTEGER, body TEXT DEFAULT 'x y' )`, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - norm := normalizeSQL(tt.sql) - if norm != tt.want { - t.Fatalf("normalizeSQL(%q) =\n %q\nwant\n %q", tt.sql, norm, tt.want) - } - }) - } -} - -func TestNormalizeSQLAdversarialCases(t *testing.T) { - // Various edge cases and adversarial inputs - tests := []struct { - name string - sql string - want string - }{ - { - name: "unterminated single quote", - sql: "CREATE TABLE t ( a TEXT DEFAULT 'x )", - want: "CREATE TABLE t ( a TEXT DEFAULT 'x )", - }, - { - name: "unterminated block comment", - sql: "CREATE TABLE t ( a INTEGER /* comment", - want: "CREATE TABLE t ( a INTEGER /* comment", - }, - { - name: "adjacent string literals", - sql: "CREATE TABLE t ( a TEXT DEFAULT 'x' 'y' )", - want: "CREATE TABLE t ( a TEXT DEFAULT 'x' 'y' )", - }, - { - name: "literal containing -- (line comment marker)", - sql: "CREATE TABLE t ( a TEXT DEFAULT 'foo -- bar' )", - want: "CREATE TABLE t ( a TEXT DEFAULT 'foo -- bar' )", - }, - { - name: "literal containing /* and */ (block comment markers)", - sql: "CREATE TABLE t ( a TEXT DEFAULT 'foo /* bar */ baz' )", - want: "CREATE TABLE t ( a TEXT DEFAULT 'foo /* bar */ baz' )", - }, - { - name: "empty string literal", - sql: "CREATE TABLE t ( a TEXT DEFAULT '' )", - want: "CREATE TABLE t ( a TEXT DEFAULT '' )", - }, - { - name: "comment containing -- marker", - sql: "CREATE TABLE t ( a INTEGER -- the -- marker )", - want: "CREATE TABLE t ( a INTEGER -- the -- marker )", - }, - { - name: "CRLF after line comment", - sql: "CREATE TABLE t ( a INTEGER -- comment\r\nb INTEGER )", - want: "CREATE TABLE t ( a INTEGER -- comment b INTEGER )", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - norm := normalizeSQL(tt.sql) - if norm != tt.want { - t.Fatalf("normalizeSQL(%q) =\n %q\nwant\n %q", tt.sql, norm, tt.want) - } - }) - } -} - -func TestNormalizeSQLCollapsesWhitespaceInsideComments(t *testing.T) { - // Comments are cosmetic — internal whitespace differences should not create - // different signatures. Two CREATE statements differing only in comment - // formatting should normalize identically (fix for latent defect #3). - tests := []struct { - name string - sql string - want string - }{ - { - name: "line comment with double space normalizes to single space", - sql: "CREATE TABLE t ( -- note double\na INTEGER )", - want: "CREATE TABLE t ( -- note double a INTEGER )", - }, - { - name: "line comment with multiple spaces", - sql: "CREATE TABLE t ( -- spaces everywhere \na INTEGER )", - want: "CREATE TABLE t ( -- spaces everywhere a INTEGER )", - }, - { - name: "block comment with double space normalizes to single space", - sql: "CREATE TABLE t ( /* note double */ a INTEGER )", - want: "CREATE TABLE t ( /* note double */ a INTEGER )", - }, - { - name: "block comment with mixed whitespace", - sql: "CREATE TABLE t ( /* multi space comment */ a INTEGER )", - want: "CREATE TABLE t ( /* multi space comment */ a INTEGER )", - }, - { - name: "inline comment inside CREATE with extra spaces", - sql: "CREATE TABLE t (\n -- inline comment\n a INTEGER\n)", - want: "CREATE TABLE t ( -- inline comment a INTEGER )", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - norm := normalizeSQL(tt.sql) - if norm != tt.want { - t.Fatalf("normalizeSQL(%q) =\n %q\nwant\n %q", tt.sql, norm, tt.want) - } - }) - } -} diff --git a/internal/compat/sql_canonical.go b/internal/compat/sql_canonical.go new file mode 100644 index 0000000..fab7b69 --- /dev/null +++ b/internal/compat/sql_canonical.go @@ -0,0 +1,179 @@ +package compat + +import ( + "strconv" + "strings" + "unicode" + "unicode/utf8" +) + +type sqlTokenKind byte + +const ( + sqlBare sqlTokenKind = iota + sqlLiteral + sqlQuotedIdentifier + sqlPunctuation +) + +type sqlToken struct { + kind sqlTokenKind + text string +} + +func canonicalSQL(input string) string { + tokens := scanSQLTokens(input) + var encoded strings.Builder + for _, token := range tokens { + encoded.WriteByte(byte('0' + token.kind)) + encoded.WriteByte(':') + encoded.WriteString(strconv.Itoa(len(token.text))) + encoded.WriteByte(':') + encoded.WriteString(token.text) + } + return encoded.String() +} + +func scanSQLTokens(input string) []sqlToken { + tokens := make([]sqlToken, 0, len(input)/2) + for i := 0; i < len(input); { + r, size := utf8.DecodeRuneInString(input[i:]) + if unicode.IsSpace(r) { + i += size + continue + } + if i+1 < len(input) && input[i] == '-' && input[i+1] == '-' { + i += 2 + for i < len(input) && input[i] != '\n' && input[i] != '\r' { + i++ + } + continue + } + if i+1 < len(input) && input[i] == '/' && input[i+1] == '*' { + _, next := scanBlockComment(input, i) + i = next + continue + } + if input[i] == '\'' { + text, next := scanSingleQuoted(input, i) + tokens = append(tokens, sqlToken{kind: sqlLiteral, text: text}) + i = next + continue + } + if input[i] == '"' || input[i] == '`' || input[i] == '[' { + close := byte('"') + doubled := true + if input[i] == '`' { + close = '`' + doubled = false + } else if input[i] == '[' { + close = ']' + doubled = false + } + text, next := scanDelimitedIdentifier(input, i, input[i], close, doubled) + tokens = append(tokens, sqlToken{kind: sqlQuotedIdentifier, text: text}) + i = next + continue + } + if op, ok := longestSQLOperator(input, i); ok { + tokens = append(tokens, sqlToken{kind: sqlPunctuation, text: op}) + i += len(op) + continue + } + start := i + for i < len(input) { + r, size = utf8.DecodeRuneInString(input[i:]) + if unicode.IsSpace(r) || isSQLPunctuation(input[i]) { + break + } + if input[i] == '\'' || input[i] == '"' || input[i] == '`' || input[i] == '[' { + break + } + if i+1 < len(input) && input[i] == '-' && input[i+1] == '-' { + break + } + if i+1 < len(input) && input[i] == '/' && input[i+1] == '*' { + break + } + i += size + } + tokens = append(tokens, sqlToken{kind: sqlBare, text: strings.ToLower(input[start:i])}) + } + return tokens +} + +func scanSingleQuoted(input string, start int) (string, int) { + var b strings.Builder + if start < len(input) { + b.WriteByte(input[start]) + } + i := start + 1 + for i < len(input) { + b.WriteByte(input[i]) + if input[i] == '\'' { + if i+1 < len(input) && input[i+1] == '\'' { + b.WriteByte(input[i+1]) + i += 2 + continue + } + i++ + return b.String(), i + } + i++ + } + return b.String(), i +} + +func scanDelimitedIdentifier(input string, start int, _ byte, close byte, doubledClose bool) (string, int) { + var b strings.Builder + if start < len(input) { + b.WriteByte(input[start]) + } + i := start + 1 + for i < len(input) { + b.WriteByte(input[i]) + if input[i] == close { + if doubledClose && i+1 < len(input) && input[i+1] == close { + b.WriteByte(input[i+1]) + i += 2 + continue + } + i++ + return b.String(), i + } + i++ + } + return b.String(), i +} + +func longestSQLOperator(input string, offset int) (string, bool) { + for _, op := range []string{"->>", "||", "->", "<<", ">>", "<=", ">=", "==", "!=", "<>"} { + if strings.HasPrefix(input[offset:], op) { + return op, true + } + } + if offset < len(input) && isSQLPunctuation(input[offset]) { + return input[offset : offset+1], true + } + return "", false +} + +func isSQLPunctuation(ch byte) bool { + switch ch { + case '(', ')', ',', ';', '.', '+', '-', '*', '/', '%', '<', '>', '=', '&', '|', '~': + return true + default: + return false + } +} + +func scanBlockComment(input string, start int) (string, int) { + i := start + 2 + for i+1 < len(input) { + if input[i] == '*' && input[i+1] == '/' { + return input[start : i+2], i + 2 + } + i++ + } + return input[start:], len(input) +} diff --git a/internal/compat/sql_canonical_test.go b/internal/compat/sql_canonical_test.go new file mode 100644 index 0000000..9506029 --- /dev/null +++ b/internal/compat/sql_canonical_test.go @@ -0,0 +1,72 @@ +package compat + +import "testing" + +func TestCanonicalSQLEquivalentRepresentationsMatch(t *testing.T) { + tests := []struct { + name string + left, right string + }{ + { + name: "punctuation whitespace", + left: "CREATE TABLE s ( a TEXT, b INTEGER )", + right: "create table s(a text,b integer)", + }, + { + name: "comments are not semantics", + left: "CREATE TABLE s (a TEXT /* historical layout */, b INTEGER)", + right: "CREATE TABLE s(a TEXT,b INTEGER)", + }, + { + name: "alter appended layout", + left: "CREATE TABLE s (a TEXT, b INTEGER\n)", + right: "CREATE TABLE s (a TEXT, b INTEGER)", + }, + { + name: "comment token boundary", + left: "CREATE TABLE s (a/**/TEXT)", + right: "CREATE TABLE s (a TEXT)", + }, + { + name: "apostrophe in line comment", + left: "CREATE TABLE t ( -- don't change lexer state\n a INTEGER, b TEXT)", + right: "CREATE TABLE t (a INTEGER,b TEXT)", + }, + { + name: "apostrophe in block comment", + left: "CREATE TABLE t ( /* don't change lexer state */ a INTEGER )", + right: "CREATE TABLE t (a INTEGER)", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got, want := canonicalSQL(tt.left), canonicalSQL(tt.right); got != want { + t.Fatalf("canonical SQL differs\nleft: %q\nright: %q", got, want) + } + }) + } +} + +func TestCanonicalSQLBehavioralDifferencesRemainDistinct(t *testing.T) { + tests := []struct { + name string + left, right string + }{ + {"literal", "CREATE TABLE s(a TEXT DEFAULT 'x y')", "CREATE TABLE s(a TEXT DEFAULT 'x y')"}, + {"literal comment marker", "CREATE TABLE s(a TEXT DEFAULT 'foo -- bar')", "CREATE TABLE s(a TEXT DEFAULT 'foo bar')"}, + {"quoted identifier", "CREATE TABLE \"a b\"(id INTEGER)", "CREATE TABLE \"a b\"(id INTEGER)"}, + {"check", "CREATE TABLE s(a INTEGER CHECK(a > 0))", "CREATE TABLE s(a INTEGER CHECK(a >= 0))"}, + {"conflict", "CREATE TABLE s(a TEXT UNIQUE)", "CREATE TABLE s(a TEXT UNIQUE ON CONFLICT REPLACE)"}, + {"deferrable", "CREATE TABLE s(a INTEGER REFERENCES p(id))", "CREATE TABLE s(a INTEGER REFERENCES p(id) DEFERRABLE)"}, + {"partial index", "CREATE INDEX i ON s(a) WHERE a > 0", "CREATE INDEX i ON s(a) WHERE a >= 0"}, + {"trigger", "CREATE TRIGGER t AFTER INSERT ON s BEGIN SELECT 1; END", "CREATE TRIGGER t AFTER INSERT ON s BEGIN SELECT 2; END"}, + {"fts tokenizer", "CREATE VIRTUAL TABLE f USING fts5(body, tokenize='porter')", "CREATE VIRTUAL TABLE f USING fts5(body, tokenize='trigram')"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if canonicalSQL(tt.left) == canonicalSQL(tt.right) { + t.Fatalf("%s unexpectedly collided", tt.name) + } + }) + } +} diff --git a/internal/compat/testdata/release-schemas/manifest.json b/internal/compat/testdata/release-schemas/manifest.json index a390ca7..4f871c4 100644 --- a/internal/compat/testdata/release-schemas/manifest.json +++ b/internal/compat/testdata/release-schemas/manifest.json @@ -6,7 +6,7 @@ "Tag": "v0.3.7", "Fixture": "v1.sql", "ProvenanceSHA256": "7b09c4c93e90b0cbd45cc0b851dcacefc0590145582886636c2e420145ccb409", - "Signature": "sha256:47b3486d2119a62b67d386a984985cd4af301576c4429ebda4afa065bc4b0ab8", + "Signature": "sha256:856b6d25b53276f480a847ce9c7ebbdf4820ef0d30857d9d074aaf0c330cc729", "AppliedVersion": 1, "HasSourceMetadata": true }, @@ -14,7 +14,7 @@ "Tag": "v0.3.9", "Fixture": "v1.sql", "ProvenanceSHA256": "7b09c4c93e90b0cbd45cc0b851dcacefc0590145582886636c2e420145ccb409", - "Signature": "sha256:47b3486d2119a62b67d386a984985cd4af301576c4429ebda4afa065bc4b0ab8", + "Signature": "sha256:856b6d25b53276f480a847ce9c7ebbdf4820ef0d30857d9d074aaf0c330cc729", "AppliedVersion": 1, "HasSourceMetadata": true }, @@ -22,7 +22,7 @@ "Tag": "v0.3.10", "Fixture": "v1.sql", "ProvenanceSHA256": "7b09c4c93e90b0cbd45cc0b851dcacefc0590145582886636c2e420145ccb409", - "Signature": "sha256:47b3486d2119a62b67d386a984985cd4af301576c4429ebda4afa065bc4b0ab8", + "Signature": "sha256:856b6d25b53276f480a847ce9c7ebbdf4820ef0d30857d9d074aaf0c330cc729", "AppliedVersion": 1, "HasSourceMetadata": true }, @@ -30,7 +30,7 @@ "Tag": "v0.3.11", "Fixture": "v3.sql", "ProvenanceSHA256": "1882e91edfc1eaf865b27b1796b8d20c92d05a2a55920f37f428df8b4e348e5d", - "Signature": "sha256:5b069c657d76eda9257eacb49da8b668560b98c075835744087556e23eaf54fc", + "Signature": "sha256:187fc156d300a6f059203a6e40826ad05fa09ee5e714af22d847c244bf25b176", "AppliedVersion": 3, "HasSourceMetadata": true }, @@ -38,7 +38,7 @@ "Tag": "v0.3.12", "Fixture": "v3.sql", "ProvenanceSHA256": "1882e91edfc1eaf865b27b1796b8d20c92d05a2a55920f37f428df8b4e348e5d", - "Signature": "sha256:5b069c657d76eda9257eacb49da8b668560b98c075835744087556e23eaf54fc", + "Signature": "sha256:187fc156d300a6f059203a6e40826ad05fa09ee5e714af22d847c244bf25b176", "AppliedVersion": 3, "HasSourceMetadata": true }, @@ -46,7 +46,7 @@ "Tag": "v0.3.13", "Fixture": "v3.sql", "ProvenanceSHA256": "1882e91edfc1eaf865b27b1796b8d20c92d05a2a55920f37f428df8b4e348e5d", - "Signature": "sha256:5b069c657d76eda9257eacb49da8b668560b98c075835744087556e23eaf54fc", + "Signature": "sha256:187fc156d300a6f059203a6e40826ad05fa09ee5e714af22d847c244bf25b176", "AppliedVersion": 3, "HasSourceMetadata": true }, @@ -54,7 +54,7 @@ "Tag": "v0.3.14", "Fixture": "v3.sql", "ProvenanceSHA256": "1882e91edfc1eaf865b27b1796b8d20c92d05a2a55920f37f428df8b4e348e5d", - "Signature": "sha256:5b069c657d76eda9257eacb49da8b668560b98c075835744087556e23eaf54fc", + "Signature": "sha256:187fc156d300a6f059203a6e40826ad05fa09ee5e714af22d847c244bf25b176", "AppliedVersion": 3, "HasSourceMetadata": true }, @@ -62,7 +62,7 @@ "Tag": "v0.3.15", "Fixture": "v3.sql", "ProvenanceSHA256": "1882e91edfc1eaf865b27b1796b8d20c92d05a2a55920f37f428df8b4e348e5d", - "Signature": "sha256:5b069c657d76eda9257eacb49da8b668560b98c075835744087556e23eaf54fc", + "Signature": "sha256:187fc156d300a6f059203a6e40826ad05fa09ee5e714af22d847c244bf25b176", "AppliedVersion": 3, "HasSourceMetadata": true }, @@ -70,7 +70,7 @@ "Tag": "v0.3.17", "Fixture": "v3.sql", "ProvenanceSHA256": "1882e91edfc1eaf865b27b1796b8d20c92d05a2a55920f37f428df8b4e348e5d", - "Signature": "sha256:5b069c657d76eda9257eacb49da8b668560b98c075835744087556e23eaf54fc", + "Signature": "sha256:187fc156d300a6f059203a6e40826ad05fa09ee5e714af22d847c244bf25b176", "AppliedVersion": 3, "HasSourceMetadata": true }, @@ -78,7 +78,7 @@ "Tag": "v0.3.18", "Fixture": "v3.sql", "ProvenanceSHA256": "1882e91edfc1eaf865b27b1796b8d20c92d05a2a55920f37f428df8b4e348e5d", - "Signature": "sha256:5b069c657d76eda9257eacb49da8b668560b98c075835744087556e23eaf54fc", + "Signature": "sha256:187fc156d300a6f059203a6e40826ad05fa09ee5e714af22d847c244bf25b176", "AppliedVersion": 3, "HasSourceMetadata": true }, @@ -86,7 +86,7 @@ "Tag": "v0.3.19", "Fixture": "v3.sql", "ProvenanceSHA256": "1882e91edfc1eaf865b27b1796b8d20c92d05a2a55920f37f428df8b4e348e5d", - "Signature": "sha256:5b069c657d76eda9257eacb49da8b668560b98c075835744087556e23eaf54fc", + "Signature": "sha256:187fc156d300a6f059203a6e40826ad05fa09ee5e714af22d847c244bf25b176", "AppliedVersion": 3, "HasSourceMetadata": true }, @@ -94,7 +94,7 @@ "Tag": "v0.4.0", "Fixture": "v3.sql", "ProvenanceSHA256": "1882e91edfc1eaf865b27b1796b8d20c92d05a2a55920f37f428df8b4e348e5d", - "Signature": "sha256:5b069c657d76eda9257eacb49da8b668560b98c075835744087556e23eaf54fc", + "Signature": "sha256:187fc156d300a6f059203a6e40826ad05fa09ee5e714af22d847c244bf25b176", "AppliedVersion": 3, "HasSourceMetadata": true }, @@ -102,7 +102,7 @@ "Tag": "v0.4.1", "Fixture": "v3.sql", "ProvenanceSHA256": "1882e91edfc1eaf865b27b1796b8d20c92d05a2a55920f37f428df8b4e348e5d", - "Signature": "sha256:5b069c657d76eda9257eacb49da8b668560b98c075835744087556e23eaf54fc", + "Signature": "sha256:187fc156d300a6f059203a6e40826ad05fa09ee5e714af22d847c244bf25b176", "AppliedVersion": 3, "HasSourceMetadata": true }, @@ -110,7 +110,7 @@ "Tag": "v0.4.3", "Fixture": "v3.sql", "ProvenanceSHA256": "1882e91edfc1eaf865b27b1796b8d20c92d05a2a55920f37f428df8b4e348e5d", - "Signature": "sha256:5b069c657d76eda9257eacb49da8b668560b98c075835744087556e23eaf54fc", + "Signature": "sha256:187fc156d300a6f059203a6e40826ad05fa09ee5e714af22d847c244bf25b176", "AppliedVersion": 3, "HasSourceMetadata": true }, @@ -118,7 +118,7 @@ "Tag": "v1.0.0", "Fixture": "v3.sql", "ProvenanceSHA256": "1882e91edfc1eaf865b27b1796b8d20c92d05a2a55920f37f428df8b4e348e5d", - "Signature": "sha256:5b069c657d76eda9257eacb49da8b668560b98c075835744087556e23eaf54fc", + "Signature": "sha256:187fc156d300a6f059203a6e40826ad05fa09ee5e714af22d847c244bf25b176", "AppliedVersion": 3, "HasSourceMetadata": true }, @@ -126,7 +126,7 @@ "Tag": "v1.0.1", "Fixture": "v3.sql", "ProvenanceSHA256": "1882e91edfc1eaf865b27b1796b8d20c92d05a2a55920f37f428df8b4e348e5d", - "Signature": "sha256:5b069c657d76eda9257eacb49da8b668560b98c075835744087556e23eaf54fc", + "Signature": "sha256:187fc156d300a6f059203a6e40826ad05fa09ee5e714af22d847c244bf25b176", "AppliedVersion": 3, "HasSourceMetadata": true }, @@ -134,7 +134,7 @@ "Tag": "v1.0.2", "Fixture": "v3.sql", "ProvenanceSHA256": "1882e91edfc1eaf865b27b1796b8d20c92d05a2a55920f37f428df8b4e348e5d", - "Signature": "sha256:5b069c657d76eda9257eacb49da8b668560b98c075835744087556e23eaf54fc", + "Signature": "sha256:187fc156d300a6f059203a6e40826ad05fa09ee5e714af22d847c244bf25b176", "AppliedVersion": 3, "HasSourceMetadata": true }, @@ -142,7 +142,7 @@ "Tag": "v1.0.3", "Fixture": "v3.sql", "ProvenanceSHA256": "1882e91edfc1eaf865b27b1796b8d20c92d05a2a55920f37f428df8b4e348e5d", - "Signature": "sha256:5b069c657d76eda9257eacb49da8b668560b98c075835744087556e23eaf54fc", + "Signature": "sha256:187fc156d300a6f059203a6e40826ad05fa09ee5e714af22d847c244bf25b176", "AppliedVersion": 3, "HasSourceMetadata": true }, @@ -150,7 +150,7 @@ "Tag": "v1.0.4", "Fixture": "v3.sql", "ProvenanceSHA256": "1882e91edfc1eaf865b27b1796b8d20c92d05a2a55920f37f428df8b4e348e5d", - "Signature": "sha256:5b069c657d76eda9257eacb49da8b668560b98c075835744087556e23eaf54fc", + "Signature": "sha256:187fc156d300a6f059203a6e40826ad05fa09ee5e714af22d847c244bf25b176", "AppliedVersion": 3, "HasSourceMetadata": true }, @@ -158,7 +158,7 @@ "Tag": "v1.1.0", "Fixture": "v3.sql", "ProvenanceSHA256": "1882e91edfc1eaf865b27b1796b8d20c92d05a2a55920f37f428df8b4e348e5d", - "Signature": "sha256:5b069c657d76eda9257eacb49da8b668560b98c075835744087556e23eaf54fc", + "Signature": "sha256:187fc156d300a6f059203a6e40826ad05fa09ee5e714af22d847c244bf25b176", "AppliedVersion": 3, "HasSourceMetadata": true }, @@ -166,7 +166,7 @@ "Tag": "v1.1.1", "Fixture": "v3.sql", "ProvenanceSHA256": "1882e91edfc1eaf865b27b1796b8d20c92d05a2a55920f37f428df8b4e348e5d", - "Signature": "sha256:5b069c657d76eda9257eacb49da8b668560b98c075835744087556e23eaf54fc", + "Signature": "sha256:187fc156d300a6f059203a6e40826ad05fa09ee5e714af22d847c244bf25b176", "AppliedVersion": 3, "HasSourceMetadata": true }, @@ -174,7 +174,7 @@ "Tag": "v1.2.0", "Fixture": "v3.sql", "ProvenanceSHA256": "1882e91edfc1eaf865b27b1796b8d20c92d05a2a55920f37f428df8b4e348e5d", - "Signature": "sha256:5b069c657d76eda9257eacb49da8b668560b98c075835744087556e23eaf54fc", + "Signature": "sha256:187fc156d300a6f059203a6e40826ad05fa09ee5e714af22d847c244bf25b176", "AppliedVersion": 3, "HasSourceMetadata": true }, @@ -182,7 +182,7 @@ "Tag": "v1.2.1", "Fixture": "v3.sql", "ProvenanceSHA256": "1882e91edfc1eaf865b27b1796b8d20c92d05a2a55920f37f428df8b4e348e5d", - "Signature": "sha256:5b069c657d76eda9257eacb49da8b668560b98c075835744087556e23eaf54fc", + "Signature": "sha256:187fc156d300a6f059203a6e40826ad05fa09ee5e714af22d847c244bf25b176", "AppliedVersion": 3, "HasSourceMetadata": true }, @@ -190,7 +190,7 @@ "Tag": "v1.3.0", "Fixture": "v3.sql", "ProvenanceSHA256": "1882e91edfc1eaf865b27b1796b8d20c92d05a2a55920f37f428df8b4e348e5d", - "Signature": "sha256:5b069c657d76eda9257eacb49da8b668560b98c075835744087556e23eaf54fc", + "Signature": "sha256:187fc156d300a6f059203a6e40826ad05fa09ee5e714af22d847c244bf25b176", "AppliedVersion": 3, "HasSourceMetadata": true }, @@ -198,7 +198,7 @@ "Tag": "v1.3.1", "Fixture": "v3.sql", "ProvenanceSHA256": "1882e91edfc1eaf865b27b1796b8d20c92d05a2a55920f37f428df8b4e348e5d", - "Signature": "sha256:5b069c657d76eda9257eacb49da8b668560b98c075835744087556e23eaf54fc", + "Signature": "sha256:187fc156d300a6f059203a6e40826ad05fa09ee5e714af22d847c244bf25b176", "AppliedVersion": 3, "HasSourceMetadata": true }, @@ -206,7 +206,7 @@ "Tag": "v1.3.2", "Fixture": "v3.sql", "ProvenanceSHA256": "1882e91edfc1eaf865b27b1796b8d20c92d05a2a55920f37f428df8b4e348e5d", - "Signature": "sha256:5b069c657d76eda9257eacb49da8b668560b98c075835744087556e23eaf54fc", + "Signature": "sha256:187fc156d300a6f059203a6e40826ad05fa09ee5e714af22d847c244bf25b176", "AppliedVersion": 3, "HasSourceMetadata": true }, @@ -214,7 +214,7 @@ "Tag": "v1.3.3", "Fixture": "v3.sql", "ProvenanceSHA256": "1882e91edfc1eaf865b27b1796b8d20c92d05a2a55920f37f428df8b4e348e5d", - "Signature": "sha256:5b069c657d76eda9257eacb49da8b668560b98c075835744087556e23eaf54fc", + "Signature": "sha256:187fc156d300a6f059203a6e40826ad05fa09ee5e714af22d847c244bf25b176", "AppliedVersion": 3, "HasSourceMetadata": true }, @@ -222,7 +222,7 @@ "Tag": "v1.3.5", "Fixture": "v3.sql", "ProvenanceSHA256": "1882e91edfc1eaf865b27b1796b8d20c92d05a2a55920f37f428df8b4e348e5d", - "Signature": "sha256:5b069c657d76eda9257eacb49da8b668560b98c075835744087556e23eaf54fc", + "Signature": "sha256:187fc156d300a6f059203a6e40826ad05fa09ee5e714af22d847c244bf25b176", "AppliedVersion": 3, "HasSourceMetadata": true }, @@ -230,7 +230,7 @@ "Tag": "v1.4.0", "Fixture": "v4.sql", "ProvenanceSHA256": "e8f28b3a1b37acd4f6bd721615c150d53c9d511c05c53e625d4de22230b3667c", - "Signature": "sha256:944c7d94376ca4de5ea321f7a8efccbddf061526cfbf556640cfae15be34f944", + "Signature": "sha256:a8b9995f969b824f58b8a710680cb7af13257b6bf21b2b975285eaa8fc20b314", "AppliedVersion": 4, "HasSourceMetadata": true }, @@ -238,7 +238,7 @@ "Tag": "v1.4.1", "Fixture": "v4.sql", "ProvenanceSHA256": "e8f28b3a1b37acd4f6bd721615c150d53c9d511c05c53e625d4de22230b3667c", - "Signature": "sha256:944c7d94376ca4de5ea321f7a8efccbddf061526cfbf556640cfae15be34f944", + "Signature": "sha256:a8b9995f969b824f58b8a710680cb7af13257b6bf21b2b975285eaa8fc20b314", "AppliedVersion": 4, "HasSourceMetadata": true }, @@ -246,7 +246,7 @@ "Tag": "v1.4.2", "Fixture": "v4.sql", "ProvenanceSHA256": "e8f28b3a1b37acd4f6bd721615c150d53c9d511c05c53e625d4de22230b3667c", - "Signature": "sha256:944c7d94376ca4de5ea321f7a8efccbddf061526cfbf556640cfae15be34f944", + "Signature": "sha256:a8b9995f969b824f58b8a710680cb7af13257b6bf21b2b975285eaa8fc20b314", "AppliedVersion": 4, "HasSourceMetadata": true }, @@ -254,7 +254,7 @@ "Tag": "v1.4.3", "Fixture": "v4.sql", "ProvenanceSHA256": "e8f28b3a1b37acd4f6bd721615c150d53c9d511c05c53e625d4de22230b3667c", - "Signature": "sha256:944c7d94376ca4de5ea321f7a8efccbddf061526cfbf556640cfae15be34f944", + "Signature": "sha256:a8b9995f969b824f58b8a710680cb7af13257b6bf21b2b975285eaa8fc20b314", "AppliedVersion": 4, "HasSourceMetadata": true }, @@ -262,7 +262,7 @@ "Tag": "v1.4.4", "Fixture": "v4.sql", "ProvenanceSHA256": "e8f28b3a1b37acd4f6bd721615c150d53c9d511c05c53e625d4de22230b3667c", - "Signature": "sha256:944c7d94376ca4de5ea321f7a8efccbddf061526cfbf556640cfae15be34f944", + "Signature": "sha256:a8b9995f969b824f58b8a710680cb7af13257b6bf21b2b975285eaa8fc20b314", "AppliedVersion": 4, "HasSourceMetadata": true }, @@ -270,7 +270,7 @@ "Tag": "v2.0.0", "Fixture": "v5-with-source-metadata.sql", "ProvenanceSHA256": "1efece456c94cf46a7f5dba8f84e0a76fb444dbd4cae016b595a7fd68427c062", - "Signature": "sha256:ccbc79ad72964f9dbbf1fef77e3530a38221c991bec7c6bec1ecd3e40bd6f3ce", + "Signature": "sha256:4b3f7f19891ddcb5a18fe787722a18a2e4ce32276e16af1aef1c635a8fdd7b3b", "AppliedVersion": 5, "HasSourceMetadata": true }, @@ -278,7 +278,7 @@ "Tag": "v2.0.1", "Fixture": "v5-with-source-metadata.sql", "ProvenanceSHA256": "1efece456c94cf46a7f5dba8f84e0a76fb444dbd4cae016b595a7fd68427c062", - "Signature": "sha256:ccbc79ad72964f9dbbf1fef77e3530a38221c991bec7c6bec1ecd3e40bd6f3ce", + "Signature": "sha256:4b3f7f19891ddcb5a18fe787722a18a2e4ce32276e16af1aef1c635a8fdd7b3b", "AppliedVersion": 5, "HasSourceMetadata": true }, @@ -286,7 +286,7 @@ "Tag": "v2.1.0", "Fixture": "v5-with-source-metadata.sql", "ProvenanceSHA256": "1efece456c94cf46a7f5dba8f84e0a76fb444dbd4cae016b595a7fd68427c062", - "Signature": "sha256:ccbc79ad72964f9dbbf1fef77e3530a38221c991bec7c6bec1ecd3e40bd6f3ce", + "Signature": "sha256:4b3f7f19891ddcb5a18fe787722a18a2e4ce32276e16af1aef1c635a8fdd7b3b", "AppliedVersion": 5, "HasSourceMetadata": true }, @@ -294,7 +294,7 @@ "Tag": "v2.2.0", "Fixture": "v7.sql", "ProvenanceSHA256": "324a876835ad39ebe9749bf1a381018c267f4c21df053589d2046d2eb9cca24e", - "Signature": "sha256:ff4f927f570680865b7f62bc9d3127b97db5ccc9c7816c4e08022a630ca79035", + "Signature": "sha256:ff1a4d3c41011b5651f6a18e9772f75ef63dae407098e13cdad037edcabc5f9e", "AppliedVersion": 7, "HasSourceMetadata": false }, @@ -302,7 +302,7 @@ "Tag": "v2.2.1", "Fixture": "v7.sql", "ProvenanceSHA256": "324a876835ad39ebe9749bf1a381018c267f4c21df053589d2046d2eb9cca24e", - "Signature": "sha256:ff4f927f570680865b7f62bc9d3127b97db5ccc9c7816c4e08022a630ca79035", + "Signature": "sha256:ff1a4d3c41011b5651f6a18e9772f75ef63dae407098e13cdad037edcabc5f9e", "AppliedVersion": 7, "HasSourceMetadata": false }, @@ -310,7 +310,7 @@ "Tag": "v2.2.2", "Fixture": "v7.sql", "ProvenanceSHA256": "324a876835ad39ebe9749bf1a381018c267f4c21df053589d2046d2eb9cca24e", - "Signature": "sha256:ff4f927f570680865b7f62bc9d3127b97db5ccc9c7816c4e08022a630ca79035", + "Signature": "sha256:ff1a4d3c41011b5651f6a18e9772f75ef63dae407098e13cdad037edcabc5f9e", "AppliedVersion": 7, "HasSourceMetadata": false }, @@ -318,7 +318,7 @@ "Tag": "v2.2.3", "Fixture": "v7.sql", "ProvenanceSHA256": "324a876835ad39ebe9749bf1a381018c267f4c21df053589d2046d2eb9cca24e", - "Signature": "sha256:ff4f927f570680865b7f62bc9d3127b97db5ccc9c7816c4e08022a630ca79035", + "Signature": "sha256:ff1a4d3c41011b5651f6a18e9772f75ef63dae407098e13cdad037edcabc5f9e", "AppliedVersion": 7, "HasSourceMetadata": false }, @@ -326,7 +326,7 @@ "Tag": "v2.3.0", "Fixture": "v8.sql", "ProvenanceSHA256": "c66577744d5feacde305382d5159b67390450c18169d3da6fda5e81051b49cf6", - "Signature": "sha256:aac78c23870733c9d3e541c8be111de99798bc8f81277beca851fbd1f465ed35", + "Signature": "sha256:e54cef3ed4dffcbae1d98edbbe91e2c4fc4b4327c52935a71a2b269dfa08ec36", "AppliedVersion": 8, "HasSourceMetadata": false }, @@ -334,7 +334,7 @@ "Tag": "v2.4.0", "Fixture": "v9.sql", "ProvenanceSHA256": "50469b7ae10e1c824b59d094290c2b59bbc31f3924175d2888bb23f76450b2a1", - "Signature": "sha256:dcd0153f47c2279a02e0c8f9505d6ac875817053de3eeb557c7ec10cf304440e", + "Signature": "sha256:786eff77ced1a5f1405dc2b79451df8aff12c4e35a0597ad57a48a76249fdbfc", "AppliedVersion": 9, "HasSourceMetadata": false }, @@ -342,7 +342,7 @@ "Tag": "v2.5.0", "Fixture": "v9.sql", "ProvenanceSHA256": "50469b7ae10e1c824b59d094290c2b59bbc31f3924175d2888bb23f76450b2a1", - "Signature": "sha256:dcd0153f47c2279a02e0c8f9505d6ac875817053de3eeb557c7ec10cf304440e", + "Signature": "sha256:786eff77ced1a5f1405dc2b79451df8aff12c4e35a0597ad57a48a76249fdbfc", "AppliedVersion": 9, "HasSourceMetadata": false }, @@ -350,7 +350,7 @@ "Tag": "v2.6.0", "Fixture": "v10.sql", "ProvenanceSHA256": "0b6fa1dbe8981705aa68e0a376c874ae7a98588380d2301d108ddcf5f4410abc", - "Signature": "sha256:95c059024be9f2b8f79f8a88bde9234bf8cc1ec238a888d6cbffce861c1db8f9", + "Signature": "sha256:2952840b7c0d01efbe606e49bcbfe26054427ce15cb47c50ac764bbb348ff8f4", "AppliedVersion": 10, "HasSourceMetadata": false }, @@ -358,7 +358,7 @@ "Tag": "v2.7.0", "Fixture": "v11.sql", "ProvenanceSHA256": "78927a394a0d1cb62651288e7626de4e2983530a3a03b9147440e56b457e26ba", - "Signature": "sha256:7ed4932adf323c57fc39a396114b9d34ed4e20b02c409fc06a6020f2516a91cd", + "Signature": "sha256:a4eba7fc52225e449522cc0f4d441723394ff5873b8909b568959acfca3749ee", "AppliedVersion": 11, "HasSourceMetadata": false }, @@ -366,7 +366,7 @@ "Tag": "v2.8.0", "Fixture": "v12.sql", "ProvenanceSHA256": "84710f15fcff04567bfe8abe64f337129722080816b7cfbce35740668aacfe41", - "Signature": "sha256:54c6f7a05268fab162fdaf1cf43412788aed90633b45ce9ce1694e1ee76b5c94", + "Signature": "sha256:621c9c2c2d004b4aededf6c7cb9ddbceaa05461e5b455997c0aa292fc2ec85f2", "AppliedVersion": 12, "HasSourceMetadata": false }, @@ -374,7 +374,7 @@ "Tag": "v2.9.0", "Fixture": "v12.sql", "ProvenanceSHA256": "84710f15fcff04567bfe8abe64f337129722080816b7cfbce35740668aacfe41", - "Signature": "sha256:54c6f7a05268fab162fdaf1cf43412788aed90633b45ce9ce1694e1ee76b5c94", + "Signature": "sha256:621c9c2c2d004b4aededf6c7cb9ddbceaa05461e5b455997c0aa292fc2ec85f2", "AppliedVersion": 12, "HasSourceMetadata": false }, @@ -382,7 +382,7 @@ "Tag": "v2.10.0", "Fixture": "v12.sql", "ProvenanceSHA256": "84710f15fcff04567bfe8abe64f337129722080816b7cfbce35740668aacfe41", - "Signature": "sha256:54c6f7a05268fab162fdaf1cf43412788aed90633b45ce9ce1694e1ee76b5c94", + "Signature": "sha256:621c9c2c2d004b4aededf6c7cb9ddbceaa05461e5b455997c0aa292fc2ec85f2", "AppliedVersion": 12, "HasSourceMetadata": false }, @@ -390,7 +390,7 @@ "Tag": "v2.11.0", "Fixture": "v12.sql", "ProvenanceSHA256": "84710f15fcff04567bfe8abe64f337129722080816b7cfbce35740668aacfe41", - "Signature": "sha256:54c6f7a05268fab162fdaf1cf43412788aed90633b45ce9ce1694e1ee76b5c94", + "Signature": "sha256:621c9c2c2d004b4aededf6c7cb9ddbceaa05461e5b455997c0aa292fc2ec85f2", "AppliedVersion": 12, "HasSourceMetadata": false }, @@ -398,7 +398,7 @@ "Tag": "v2.12.0", "Fixture": "v13.sql", "ProvenanceSHA256": "9d9e23de5a05318f1f3625b0d375b64b10e42672ddd213f84a08c4375c0afdcc", - "Signature": "sha256:5d6baa68ddeb058a293aa80fce9371098c904b6ebb15321eb5fb5bef5c045adc", + "Signature": "sha256:bc02e80bddf32a68341c7bb5f28d6b1b95b3a6877436478743ae260541494b3e", "AppliedVersion": 13, "HasSourceMetadata": false }, @@ -406,7 +406,7 @@ "Tag": "v2.13.0", "Fixture": "v13.sql", "ProvenanceSHA256": "9d9e23de5a05318f1f3625b0d375b64b10e42672ddd213f84a08c4375c0afdcc", - "Signature": "sha256:5d6baa68ddeb058a293aa80fce9371098c904b6ebb15321eb5fb5bef5c045adc", + "Signature": "sha256:bc02e80bddf32a68341c7bb5f28d6b1b95b3a6877436478743ae260541494b3e", "AppliedVersion": 13, "HasSourceMetadata": false }, @@ -414,7 +414,7 @@ "Tag": "v2.14.0", "Fixture": "v13.sql", "ProvenanceSHA256": "9d9e23de5a05318f1f3625b0d375b64b10e42672ddd213f84a08c4375c0afdcc", - "Signature": "sha256:5d6baa68ddeb058a293aa80fce9371098c904b6ebb15321eb5fb5bef5c045adc", + "Signature": "sha256:bc02e80bddf32a68341c7bb5f28d6b1b95b3a6877436478743ae260541494b3e", "AppliedVersion": 13, "HasSourceMetadata": false }, @@ -422,7 +422,7 @@ "Tag": "v2.14.1", "Fixture": "v13.sql", "ProvenanceSHA256": "9d9e23de5a05318f1f3625b0d375b64b10e42672ddd213f84a08c4375c0afdcc", - "Signature": "sha256:5d6baa68ddeb058a293aa80fce9371098c904b6ebb15321eb5fb5bef5c045adc", + "Signature": "sha256:bc02e80bddf32a68341c7bb5f28d6b1b95b3a6877436478743ae260541494b3e", "AppliedVersion": 13, "HasSourceMetadata": false }, @@ -430,7 +430,7 @@ "Tag": "v2.14.2", "Fixture": "v13.sql", "ProvenanceSHA256": "9d9e23de5a05318f1f3625b0d375b64b10e42672ddd213f84a08c4375c0afdcc", - "Signature": "sha256:5d6baa68ddeb058a293aa80fce9371098c904b6ebb15321eb5fb5bef5c045adc", + "Signature": "sha256:bc02e80bddf32a68341c7bb5f28d6b1b95b3a6877436478743ae260541494b3e", "AppliedVersion": 13, "HasSourceMetadata": false }, @@ -438,7 +438,7 @@ "Tag": "v2.14.3", "Fixture": "v13.sql", "ProvenanceSHA256": "9d9e23de5a05318f1f3625b0d375b64b10e42672ddd213f84a08c4375c0afdcc", - "Signature": "sha256:5d6baa68ddeb058a293aa80fce9371098c904b6ebb15321eb5fb5bef5c045adc", + "Signature": "sha256:bc02e80bddf32a68341c7bb5f28d6b1b95b3a6877436478743ae260541494b3e", "AppliedVersion": 13, "HasSourceMetadata": false }, @@ -446,7 +446,7 @@ "Tag": "v2.15.0", "Fixture": "v13.sql", "ProvenanceSHA256": "9d9e23de5a05318f1f3625b0d375b64b10e42672ddd213f84a08c4375c0afdcc", - "Signature": "sha256:5d6baa68ddeb058a293aa80fce9371098c904b6ebb15321eb5fb5bef5c045adc", + "Signature": "sha256:bc02e80bddf32a68341c7bb5f28d6b1b95b3a6877436478743ae260541494b3e", "AppliedVersion": 13, "HasSourceMetadata": false }, @@ -454,7 +454,7 @@ "Tag": "v2.15.1", "Fixture": "v13.sql", "ProvenanceSHA256": "9d9e23de5a05318f1f3625b0d375b64b10e42672ddd213f84a08c4375c0afdcc", - "Signature": "sha256:5d6baa68ddeb058a293aa80fce9371098c904b6ebb15321eb5fb5bef5c045adc", + "Signature": "sha256:bc02e80bddf32a68341c7bb5f28d6b1b95b3a6877436478743ae260541494b3e", "AppliedVersion": 13, "HasSourceMetadata": false }, @@ -462,7 +462,7 @@ "Tag": "v2.16.0", "Fixture": "v13.sql", "ProvenanceSHA256": "9d9e23de5a05318f1f3625b0d375b64b10e42672ddd213f84a08c4375c0afdcc", - "Signature": "sha256:5d6baa68ddeb058a293aa80fce9371098c904b6ebb15321eb5fb5bef5c045adc", + "Signature": "sha256:bc02e80bddf32a68341c7bb5f28d6b1b95b3a6877436478743ae260541494b3e", "AppliedVersion": 13, "HasSourceMetadata": false }, @@ -470,7 +470,7 @@ "Tag": "v2.16.1", "Fixture": "v13.sql", "ProvenanceSHA256": "9d9e23de5a05318f1f3625b0d375b64b10e42672ddd213f84a08c4375c0afdcc", - "Signature": "sha256:5d6baa68ddeb058a293aa80fce9371098c904b6ebb15321eb5fb5bef5c045adc", + "Signature": "sha256:bc02e80bddf32a68341c7bb5f28d6b1b95b3a6877436478743ae260541494b3e", "AppliedVersion": 13, "HasSourceMetadata": false }, @@ -478,7 +478,7 @@ "Tag": "v3.0.0", "Fixture": "v13.sql", "ProvenanceSHA256": "9d9e23de5a05318f1f3625b0d375b64b10e42672ddd213f84a08c4375c0afdcc", - "Signature": "sha256:5d6baa68ddeb058a293aa80fce9371098c904b6ebb15321eb5fb5bef5c045adc", + "Signature": "sha256:bc02e80bddf32a68341c7bb5f28d6b1b95b3a6877436478743ae260541494b3e", "AppliedVersion": 13, "HasSourceMetadata": false }, @@ -486,7 +486,7 @@ "Tag": "v3.0.1", "Fixture": "v13.sql", "ProvenanceSHA256": "9d9e23de5a05318f1f3625b0d375b64b10e42672ddd213f84a08c4375c0afdcc", - "Signature": "sha256:5d6baa68ddeb058a293aa80fce9371098c904b6ebb15321eb5fb5bef5c045adc", + "Signature": "sha256:bc02e80bddf32a68341c7bb5f28d6b1b95b3a6877436478743ae260541494b3e", "AppliedVersion": 13, "HasSourceMetadata": false }, @@ -494,7 +494,7 @@ "Tag": "v3.0.2", "Fixture": "v13.sql", "ProvenanceSHA256": "9d9e23de5a05318f1f3625b0d375b64b10e42672ddd213f84a08c4375c0afdcc", - "Signature": "sha256:5d6baa68ddeb058a293aa80fce9371098c904b6ebb15321eb5fb5bef5c045adc", + "Signature": "sha256:bc02e80bddf32a68341c7bb5f28d6b1b95b3a6877436478743ae260541494b3e", "AppliedVersion": 13, "HasSourceMetadata": false }, @@ -502,7 +502,7 @@ "Tag": "v3.1.0", "Fixture": "v13.sql", "ProvenanceSHA256": "9d9e23de5a05318f1f3625b0d375b64b10e42672ddd213f84a08c4375c0afdcc", - "Signature": "sha256:5d6baa68ddeb058a293aa80fce9371098c904b6ebb15321eb5fb5bef5c045adc", + "Signature": "sha256:bc02e80bddf32a68341c7bb5f28d6b1b95b3a6877436478743ae260541494b3e", "AppliedVersion": 13, "HasSourceMetadata": false }, @@ -510,7 +510,7 @@ "Tag": "v3.2.0", "Fixture": "v13.sql", "ProvenanceSHA256": "9d9e23de5a05318f1f3625b0d375b64b10e42672ddd213f84a08c4375c0afdcc", - "Signature": "sha256:5d6baa68ddeb058a293aa80fce9371098c904b6ebb15321eb5fb5bef5c045adc", + "Signature": "sha256:bc02e80bddf32a68341c7bb5f28d6b1b95b3a6877436478743ae260541494b3e", "AppliedVersion": 13, "HasSourceMetadata": false }, @@ -518,7 +518,7 @@ "Tag": "v3.2.1", "Fixture": "v13.sql", "ProvenanceSHA256": "9d9e23de5a05318f1f3625b0d375b64b10e42672ddd213f84a08c4375c0afdcc", - "Signature": "sha256:5d6baa68ddeb058a293aa80fce9371098c904b6ebb15321eb5fb5bef5c045adc", + "Signature": "sha256:bc02e80bddf32a68341c7bb5f28d6b1b95b3a6877436478743ae260541494b3e", "AppliedVersion": 13, "HasSourceMetadata": false }, @@ -526,7 +526,7 @@ "Tag": "v3.2.2", "Fixture": "v13.sql", "ProvenanceSHA256": "9d9e23de5a05318f1f3625b0d375b64b10e42672ddd213f84a08c4375c0afdcc", - "Signature": "sha256:5d6baa68ddeb058a293aa80fce9371098c904b6ebb15321eb5fb5bef5c045adc", + "Signature": "sha256:bc02e80bddf32a68341c7bb5f28d6b1b95b3a6877436478743ae260541494b3e", "AppliedVersion": 13, "HasSourceMetadata": false }, @@ -534,7 +534,7 @@ "Tag": "v3.2.3", "Fixture": "v13.sql", "ProvenanceSHA256": "9d9e23de5a05318f1f3625b0d375b64b10e42672ddd213f84a08c4375c0afdcc", - "Signature": "sha256:5d6baa68ddeb058a293aa80fce9371098c904b6ebb15321eb5fb5bef5c045adc", + "Signature": "sha256:bc02e80bddf32a68341c7bb5f28d6b1b95b3a6877436478743ae260541494b3e", "AppliedVersion": 13, "HasSourceMetadata": false }, @@ -542,7 +542,7 @@ "Tag": "v3.2.4", "Fixture": "v13.sql", "ProvenanceSHA256": "9d9e23de5a05318f1f3625b0d375b64b10e42672ddd213f84a08c4375c0afdcc", - "Signature": "sha256:5d6baa68ddeb058a293aa80fce9371098c904b6ebb15321eb5fb5bef5c045adc", + "Signature": "sha256:bc02e80bddf32a68341c7bb5f28d6b1b95b3a6877436478743ae260541494b3e", "AppliedVersion": 13, "HasSourceMetadata": false }, @@ -550,7 +550,7 @@ "Tag": "v3.2.5", "Fixture": "v13.sql", "ProvenanceSHA256": "9d9e23de5a05318f1f3625b0d375b64b10e42672ddd213f84a08c4375c0afdcc", - "Signature": "sha256:5d6baa68ddeb058a293aa80fce9371098c904b6ebb15321eb5fb5bef5c045adc", + "Signature": "sha256:bc02e80bddf32a68341c7bb5f28d6b1b95b3a6877436478743ae260541494b3e", "AppliedVersion": 13, "HasSourceMetadata": false } @@ -559,7 +559,7 @@ { "Fixture": "v2.sql", "ProvenanceSHA256": "9e5857f14e30fe1391445064bf924b0efb12fa81d0e5011de72ce5f880cd7f6a", - "Signature": "sha256:16da30dbb7692750430b6ba37754a9cf42bcaec1f04507da949f8adae01442c1", + "Signature": "sha256:04ef998cbafc2be61a660b4811c33a477e973f13be8a041e9f238a94a2061009", "AppliedVersion": 2, "HasSourceMetadata": true, "Provenance": "No manifest release tag maps to this V2-only compatibility triangulation fixture." @@ -567,7 +567,7 @@ { "Fixture": "v3-no-source-metadata.sql", "ProvenanceSHA256": "42013f814ed39dbbcf8bc8e6465777d628a8d38694713f59caaeb30e89f28f68", - "Signature": "sha256:37d4cebbdcf6c5d43d2bf58b28a864c9434943709bd0975037377be6f1de1082", + "Signature": "sha256:6fe905d0141a017b9eab93092ececc14424e3be7b42568bdf0f62748859a7f86", "AppliedVersion": 3, "HasSourceMetadata": false, "Provenance": "No manifest release tag maps to this partially migrated V3 compatibility triangulation fixture." @@ -575,7 +575,7 @@ { "Fixture": "v5-without-source-metadata.sql", "ProvenanceSHA256": "fb705645b2f77017f1e9f512ba32246cefb9981cde76f78cd929f2659fdfee84", - "Signature": "sha256:f9aebeb59d28378d0a08737123ceb1fcb29e85aa374cb5445f9a73411649d1db", + "Signature": "sha256:537d42b8f4b56e16cb4fa9a85c6dba8767e377fa4ac38682f6e4613a398bf647", "AppliedVersion": 5, "HasSourceMetadata": false, "Provenance": "No manifest release tag maps to this partially migrated V5 compatibility triangulation fixture." @@ -583,7 +583,7 @@ { "Fixture": "v6.sql", "ProvenanceSHA256": "a42b23b541cc83d34ebd5571f0d7ef7771a540fe3d7e71b1a3e537a5e5932af0", - "Signature": "sha256:d88079c35be727d03874f8e827655c5e6568df369d528338d0f77aecceef5e09", + "Signature": "sha256:537d42b8f4b56e16cb4fa9a85c6dba8767e377fa4ac38682f6e4613a398bf647", "AppliedVersion": 6, "HasSourceMetadata": false, "Provenance": "No manifest release tag maps to this partial legacy per-step V6-before-V7 lineage fixture; no release shipped V6 alone." @@ -591,7 +591,7 @@ { "Fixture": "v13-development-alter-built.sql", "ProvenanceSHA256": "20766795c6e1cc8196310f9146298161791aef9ba6fbc654753ec2c1cd2fed7d", - "Signature": "sha256:4d04377754986f0da2f61f2ab73889168f596eb84e1f02df96e23072072e9375", + "Signature": "sha256:bc02e80bddf32a68341c7bb5f28d6b1b95b3a6877436478743ae260541494b3e", "AppliedVersion": 13, "HasSourceMetadata": false, "Provenance": "Explicit observed pre-release V13 development lineage produced by the historical ALTER migration path with the original V9 checksum; schema-only fixture with no user data." @@ -599,7 +599,7 @@ { "Fixture": "v13-legacy-alter-built.sql", "ProvenanceSHA256": "c592cfec531fc37577a6deddfae7537157406bda1d2a7e890398c0aeacd6d505", - "Signature": "sha256:6003ed9f20a8379761367f559c7a2e0405a9b56429044d672481b2cce23210ff", + "Signature": "sha256:bc02e80bddf32a68341c7bb5f28d6b1b95b3a6877436478743ae260541494b3e", "AppliedVersion": 13, "HasSourceMetadata": false, "Provenance": "Explicit known legacy V13 shape reproduced from the ALTER-built current schema produced by the historical migration chain; supported despite conservative full-DDL signing." @@ -607,7 +607,7 @@ { "Fixture": "v13-legacy-existing-schema-migrations.sql", "ProvenanceSHA256": "75d9463f55287704fd861f0147b0b63a6f73c5252f55a561c848dbf09c86b6f0", - "Signature": "sha256:5d6baa68ddeb058a293aa80fce9371098c904b6ebb15321eb5fb5bef5c045adc", + "Signature": "sha256:bc02e80bddf32a68341c7bb5f28d6b1b95b3a6877436478743ae260541494b3e", "AppliedVersion": 13, "HasSourceMetadata": false, "Provenance": "Explicit known legacy V13 shape produced when compatibility migrations start from a database whose schema_migrations table predated the current SetupSchema formatting; supported despite conservative full-DDL signing." @@ -615,7 +615,7 @@ { "Fixture": "v14.sql", "ProvenanceSHA256": "10ff327432c2ff5a6072a8bbd7fac6f5d864382766c40a9d31ffa3eb66a8bfc1", - "Signature": "sha256:e4973d300c6a89a5d04b510efdd9708678770fbef7e42285f5bc51c4645d85c2", + "Signature": "sha256:0b178c82e778520ff8b92d2a51e2692384845cc2a6e342f05f084c01b35fe0cb", "AppliedVersion": 14, "HasSourceMetadata": false, "Provenance": "Migration v14 adds file_size and file_mtime columns to indexed_files for startup prefilter optimization." diff --git a/internal/storage/migration_plan_test.go b/internal/storage/migration_plan_test.go index 8e274b4..eae6b9a 100644 --- a/internal/storage/migration_plan_test.go +++ b/internal/storage/migration_plan_test.go @@ -14,7 +14,7 @@ import ( "github.com/pablontiv/backscroll/internal/compat" ) -func TestCatalogGoLineagesUpgradeLosslessly(t *testing.T) { +func TestEveryCatalogFixtureReachesCurrentSemanticHead(t *testing.T) { catalog, err := compat.LoadCatalog() if err != nil { t.Fatal(err) @@ -62,14 +62,21 @@ func TestCatalogGoLineagesUpgradeLosslessly(t *testing.T) { }) } + required := []string{ + "v1.sql", "v2.sql", "v3.sql", "v3-no-source-metadata.sql", + "v4.sql", "v5-with-source-metadata.sql", "v5-without-source-metadata.sql", + "v6.sql", "v7.sql", "v8.sql", "v9.sql", "v10.sql", "v11.sql", "v12.sql", + "v13.sql", "v13-legacy-existing-schema-migrations.sql", + "v13-legacy-alter-built.sql", "v13-development-alter-built.sql", "v14.sql", + } + for _, name := range required { + if !seen[name] { + t.Fatalf("catalog closure corpus lacks %s", name) + } + } + for _, fixture := range fixtures { t.Run(fixture.name, func(t *testing.T) { - // Legacy development/alter-built v13 fixtures have special index ordering that doesn't - // produce canonical v14 schemas. These are v13 compatibility test cases, not upgrade targets. - if fixture.name == "v13-development-alter-built.sql" || fixture.name == "v13-legacy-alter-built.sql" { - t.Skip("legacy v13 ALTER-built fixtures are v13 compatibility cases, not v14 upgrade targets") - } - dbPath := createFixtureDatabase(t, fixture.name) want := seedFixtureSentinels(t, dbPath) @@ -83,9 +90,18 @@ func TestCatalogGoLineagesUpgradeLosslessly(t *testing.T) { assertSearchItemsByUUID(t, db.DB(), want.ToolSearchItems) assertTableSentinels(t, db.DB(), want) assertFTSQueryable(t, db.DB(), "sentinelterm", len(want.SearchItems)) - assertFTSQueryable(t, db.DB(), "sentinelcmd", 0) assertToolFTSQueryable(t, db.DB(), "sentinelcmd", len(want.ToolSearchItems)) assertCurrentShape(t, db.DB()) + plan, diag, err := compat.InspectIndex(context.Background(), db.DB()) + if err != nil || diag != nil { + t.Fatalf("inspect current head error=%v diagnostic=%+v", err, diag) + } + if len(plan.Steps) != 0 { + t.Fatalf("fixture %s retained steps: %+v", fixture.name, plan.Steps) + } + if plan.From != catalog.CurrentShape() { + t.Fatalf("fixture %s reached shape %+v, want %+v", fixture.name, plan.From, catalog.CurrentShape()) + } wantMigrationRows := authoritativeCurrentMigrationRows() if fixture.name == "v13-development-alter-built.sql" { for i := range wantMigrationRows { @@ -1536,44 +1552,3 @@ func onlySnapshot(t *testing.T, dbPath string) string { } return matches[0] } - -// TestMigratedFixtureSignatureIsInCatalog is a regression test for issue #52. -// It verifies that when a V1 fixture is migrated forward through the real -// SetupSchema() code, the resulting database schema signature is recognized by -// the catalog. This prevents cosmetic DDL formatting differences from silently -// ejecting valid schemas from the lineage catalog. -// -// Before the fix to normalizeSQL() (making it whitespace-insensitive), databases -// that were created at V1 and then migrated V8→V13 by published releases would -// produce a signature not in the catalog, causing every operational command to -// reject the database as "unsupported_lineage". This test would have caught that -// regression during development. -func TestMigratedFixtureSignatureIsInCatalog(t *testing.T) { - // Load v1.sql, the earliest released version fixture - dbPath := createFixtureDatabase(t, "v1.sql") - - // Open the fixture and trigger SetupSchema to migrate it all the way to V13 - db, diag, err := OpenCompatible(context.Background(), dbPath) - if err != nil || diag != nil { - t.Fatalf("open compatible error=%v diagnostic=%+v", err, diag) - } - defer func() { _ = db.Close() }() - - // Inspect the resulting schema to get its signature - plan, diag, err := compat.InspectIndex(context.Background(), db.DB()) - if err != nil || diag != nil { - t.Fatalf("inspect index error=%v diagnostic=%+v", err, diag) - } - - // Load the lineage catalog - catalog, err := compat.LoadCatalog() - if err != nil { - t.Fatal(err) - } - - // Assert that the migrated database signature is in the catalog - if !catalog.IsKnownSignature(plan.From.Signature) { - t.Errorf("migrated V1→V13 database has signature %s not in catalog; this was the bug in issue #52", - plan.From.Signature) - } -} diff --git a/internal/storage/recovery_records.go b/internal/storage/recovery_records.go index b92f8cd..bf8862c 100644 --- a/internal/storage/recovery_records.go +++ b/internal/storage/recovery_records.go @@ -26,23 +26,23 @@ func ReadRecoveryInputFromQueryer(ctx context.Context, q compat.Queryer) (compat return compat.RecoveryInput{}, diag, err } - records, diag, err := readRecordsForSignature(ctx, q, plan.From.Signature) + records, diag, err := readRecordsForShape(ctx, q, plan.From) if err != nil || diag != nil { return compat.RecoveryInput{}, diag, err } return compat.RecoveryInput{Shape: plan.From, Records: records, RowCount: len(records)}, nil, nil } -func readRecordsForSignature(ctx context.Context, q compat.Queryer, signature string) ([]models.IndexedRecord, *compat.Diagnostic, error) { +func readRecordsForShape(ctx context.Context, q compat.Queryer, shape compat.SchemaShape) ([]models.IndexedRecord, *compat.Diagnostic, error) { catalog, err := compat.LoadCatalog() if err != nil { return nil, nil, fmt.Errorf("load lineage catalog: %w", err) } - if !catalog.IsKnownSignature(signature) { + if !catalog.IsKnownShape(shape) { return nil, &compat.Diagnostic{ Code: compat.CodeUnsupportedLineage, - Summary: fmt.Sprintf("unsupported index schema %s", signature), + Summary: fmt.Sprintf("unsupported index schema version %d signature %s", shape.AppliedVersion, shape.Signature), }, nil } diff --git a/internal/storage/recovery_records_test.go b/internal/storage/recovery_records_test.go index c02e269..e730bb7 100644 --- a/internal/storage/recovery_records_test.go +++ b/internal/storage/recovery_records_test.go @@ -115,6 +115,28 @@ func TestReadRecoveryInputRejectsUnknownShape(t *testing.T) { assertRecoveryFixtureUnchanged(t, handle) } +func TestReadRecordsForShapeRejectsKnownSignatureWithUnknownVersion(t *testing.T) { + dbPath := createFixtureDatabase(t, "v14.sql") + db, err := OpenReadOnly(dbPath) + if err != nil { + t.Fatal(err) + } + defer db.Close() + catalog, err := compat.LoadCatalog() + if err != nil { + t.Fatal(err) + } + shape := catalog.CurrentShape() + shape.AppliedVersion++ + records, diag, err := readRecordsForShape(context.Background(), db.DB(), shape) + if err != nil { + t.Fatal(err) + } + if records != nil || diag == nil || diag.Code != compat.CodeUnsupportedLineage { + t.Fatalf("records=%v diagnostic=%+v", records, diag) + } +} + func TestReadRecoveryInputRejectsMissingCanonicalPayload(t *testing.T) { dbPath := buildRecoveryFixtureDatabase(t, "active-v13.sql") mutateRecoveryDatabase(t, dbPath, `