fix(etl): never pair a full derive with a partial catch-up window

docs/etl.md

@@ -116,12 +116,27 @@ node scripts/load-eop.mjs --from=YYYY-MM-DD --to=YYYY-MM-DD --no-ocds
 Първоначалният backfill и ежедневният refresh ползват едни и същи staging таблици, mapper-и и
 SQL. Различават се по прозореца от дати и режима на derive:
 
-- **голямо или първоначално догонване:** CLI прозорец + пълен derive;
-- **малък steady-state refresh:** gap-aware прозорец + slice derive.
+- **първоначално зареждане или пълно презареждане:** прозорец от началото на емисията + пълен derive;
+- **догонване и steady-state refresh:** gap-aware прозорец + slice derive.
 
 `--derive=full` пуска amendment rollup, FX, NUTS, пълна нормализация и precompute. `--derive=slice`
-пуска scoped refresh SQL-а. По подразбиране catch-up логиката избира full derive за големи
-празнини и slice derive за малки.
+пуска scoped refresh SQL-а.
+
+Пълният derive **презижда** доменните таблици от staging — `normalize-raw.sql` започва с
+`DELETE FROM contracts` — тоест всичко извън заредения прозорец отпада и не се връща. Затова той е
+допустим само когато прозорецът стига до началото на емисията (или когато още няма корпус). Понеже
+gap-aware прозорецът по устройство покрива само опашката, `--catchup` върху **вече зареден** корпус
+прави slice derive, колкото и голяма да е празнината. Изключението е първото пускане: когато няма
+никакви заредени дни, догонването взима прозорец от началото на емисията и пълен derive — там няма
+какво да се загуби.
+
+Пълен derive с частичен прозорец върху вече зареден корпус `import.mjs` отказва да продължи — преди
+зареждането — вместо да изтрие историята. Отказът важи за **действащия** режим, не само за изрично
+подадения: без `--catchup` подразбиращият се derive е `full`, тъй че и `import.mjs --from=2026-06-01`
+получава същия отказ. Проверката пита за всяка таблица, която `normalize-raw.sql` изпразва (виж
+`@full-clear` там), не само за `contracts` — корпус без договори, но с попълнени `tenders` или
+`bidders` е точно състоянието, което половинчат пробег оставя. Ако някоя от тях не може да бъде
+прочетена, отказът пак важи: проверка, която не може да провери, не бива да пуска нататък.
 
 ## Доменна нормализация (derive)
 

packages/ingest/src/ocds.test.ts

@@ -2,6 +2,7 @@ import { describe, expect, it } from 'vitest';
 import {
   classifyBucketKey,
   computeCatchupWindow,
+  fullDeriveIsSafe,
   releaseToAmendments,
   releaseToContracts,
   releaseToLots,
@@ -352,6 +353,27 @@ describe('bucket key and catchup helpers', () => {
       computeCatchupWindow({ maxLoadedDate: '2026-06-01', today: '2026-06-07', lookbackDays: 3 }),
     ).toEqual({ from: '2026-05-29', to: '2026-06-07' });
   });
+
+  it('allows a full derive only when the window reaches the start of the feed', () => {
+    // Initial backfill: nothing to lose, any window is safe.
+    expect(
+      fullDeriveIsSafe({ windowFrom: '2026-06-10', feedStart: '2020-01-01', hasCorpus: false }),
+    ).toBe(true);
+    // Whole feed reloaded into staging — the rebuild is complete.
+    expect(
+      fullDeriveIsSafe({ windowFrom: '2020-01-01', feedStart: '2020-01-01', hasCorpus: true }),
+    ).toBe(true);
+    // A catch-up window over an existing corpus would drop everything before it.
+    expect(
+      fullDeriveIsSafe({ windowFrom: '2026-06-10', feedStart: '2020-01-01', hasCorpus: true }),
+    ).toBe(false);
+  });
+
+  it('rejects malformed days rather than silently allowing a full derive', () => {
+    expect(() =>
+      fullDeriveIsSafe({ windowFrom: '10-06-2026', feedStart: '2020-01-01', hasCorpus: true }),
+    ).toThrow(/windowFrom/);
+  });
 });
 
 describe('splitSqlStatements', () => {

packages/ingest/src/ocds.ts

@@ -452,6 +452,27 @@ export function computeCatchupWindow({
   return { from: from > today ? today : from, to: today };
 }
 
+/**
+ * A full derive rebuilds the domain from whatever the staging tables hold — `scripts/normalize-raw.sql`
+ * opens with `DELETE FROM contracts` — so every contract outside the loaded window is dropped. It is
+ * only sound when the window reaches back to the first day the feed is loaded from, or when there is
+ * no corpus yet (the initial backfill). A gap-aware catch-up window never does, which is why
+ * `--catchup` derives a slice.
+ */
+export function fullDeriveIsSafe({
+  windowFrom,
+  feedStart,
+  hasCorpus,
+}: {
+  windowFrom: string;
+  feedStart: string;
+  hasCorpus: boolean;
+}): boolean {
+  validateDay(windowFrom, 'windowFrom');
+  validateDay(feedStart, 'feedStart');
+  return !hasCorpus || windowFrom <= feedStart;
+}
+
 export function daysInWindow(from: string, to: string): number {
   validateDay(from, 'from');
   validateDay(to, 'to');

packages/ingest/src/refresh-full-clear.test.ts

@@ -0,0 +1,77 @@
+/// <reference types="node" />
+import { readFileSync } from 'node:fs';
+import { dirname, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { describe, expect, it } from 'vitest';
+import { fullClearTables } from './refresh';
+
+// fullClearTables answers one question for scripts/import.mjs: which tables does a full derive empty,
+// and therefore what does a partial window destroy? The guard that used to ask it named `contracts`
+// alone while the clear had reached fourteen tables — so the tests that matter here are the ones that
+// keep the answer tied to the SQL rather than to a copy of it.
+
+const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..');
+const normalizeRaw = readFileSync(resolve(root, 'scripts/normalize-raw.sql'), 'utf8');
+
+describe('fullClearTables', () => {
+  it('reads the block out of the real normalize-raw.sql', () => {
+    const tables = fullClearTables(normalizeRaw);
+    // The base domain tables: whatever else the block grows, losing any of these is losing the corpus.
+    expect(tables).toEqual(
+      expect.arrayContaining(['contracts', 'lots', 'tenders', 'bidders', 'authorities']),
+    );
+  });
+
+  it('stops before the per-run metadata resets further down the file', () => {
+    // normalize-raw.sql also clears data_freshness and pipeline_stats, which are rewritten every run.
+    // Counting them as corpus would make the guard refuse EVERY full derive, initial backfill included
+    // — a guard that always refuses gets deleted, so this boundary is load-bearing.
+    const tables = fullClearTables(normalizeRaw);
+    expect(tables).not.toContain('data_freshness');
+    expect(tables).not.toContain('pipeline_stats');
+  });
+
+  it('keeps the marker and the block adjacent', () => {
+    // If the marker is dropped or drifts away from the DELETEs, this returns [] and import.mjs throws
+    // rather than silently deciding the corpus is empty and letting the destructive path through.
+    expect(fullClearTables(normalizeRaw).length).toBeGreaterThanOrEqual(5);
+  });
+
+  it('takes only the marked block, and only unqualified deletes', () => {
+    const sql = [
+      'DELETE FROM before_the_marker;',
+      '-- @full-clear',
+      'DROP TABLE IF EXISTS scratch;',
+      'DELETE FROM contracts;',
+      "DELETE FROM lots WHERE id = 'x';", // scoped: not a full clear
+      'DELETE FROM authorities;',
+      '',
+      'DELETE FROM after_the_block;',
+    ].join('\n');
+    expect(fullClearTables(sql)).toEqual(['contracts', 'authorities']);
+  });
+
+  it('sees a table however it is quoted', () => {
+    // `DELETE FROM "search_index";` is valid SQLite and reads as pure formatting. A bare-identifier
+    // matcher dropped it from the list, which silently reopened the data-loss hole: the guard would
+    // then wave through a corpus whose only populated table was the re-quoted one.
+    const sql = [
+      '-- @full-clear',
+      'DELETE FROM "search_index";',
+      'DELETE FROM `flow_pairs`;',
+      'DELETE FROM [home_totals];',
+      'delete from contracts;',
+      '',
+    ].join('\n');
+    expect(fullClearTables(sql)).toEqual([
+      'search_index',
+      'flow_pairs',
+      'home_totals',
+      'contracts',
+    ]);
+  });
+
+  it('returns nothing when the marker is absent', () => {
+    expect(fullClearTables('DELETE FROM contracts;\nDELETE FROM lots;\n')).toEqual([]);
+  });
+});

packages/ingest/src/refresh.ts

@@ -108,6 +108,45 @@ export function transientStagingStatements(workStagingSchemaSql: string): string
   );
 }
 
+const FULL_CLEAR_MARKER = /^--\s*@full-clear\b/i;
+// All three SQLite quoting styles, not just the bare identifier. Rewriting one line as
+// `DELETE FROM "search_index";` is a valid, invisible formatting change — and with a bare-only
+// matcher it would drop that table out of the guard's list and quietly reopen the hole this parser
+// exists to close.
+const DELETE_FROM =
+  /^DELETE\s+FROM\s+(?:"([^"]+)"|`([^`]+)`|\[([^\]]+)\]|([A-Za-z_][A-Za-z0-9_]*))\s*;?\s*$/i;
+
+/**
+ * The tables `scripts/normalize-raw.sql` empties before rebuilding the domain from staging, read out
+ * of the SQL rather than restated in JS. The guard that consumes this list used to ask about
+ * `contracts` alone while the clear had grown to fourteen tables — a hardcoded copy of a destructive
+ * list is a data-loss bug on a timer, so the list has exactly one home.
+ *
+ * Scoped to the `@full-clear` block on purpose: the same file later resets `data_freshness` and
+ * `pipeline_stats`, which are per-run metadata. Counting those as corpus would make the guard refuse
+ * every full derive, including the initial backfill it is supposed to let through.
+ */
+export function fullClearTables(normalizeRawSql: string): string[] {
+  const tables: string[] = [];
+  let inBlock = false;
+  for (const line of normalizeRawSql.split(/\r?\n/)) {
+    const trimmed = line.trim();
+    if (FULL_CLEAR_MARKER.test(trimmed)) {
+      inBlock = true;
+      continue;
+    }
+    if (!inBlock) continue;
+    const hit = trimmed.match(DELETE_FROM);
+    if (hit) {
+      tables.push((hit[1] ?? hit[2] ?? hit[3] ?? hit[4])!);
+      continue;
+    }
+    // Comments and the DROP TABLEs share the block; a blank line ends it.
+    if (trimmed === '') break;
+  }
+  return tables;
+}
+
 export function dropTransientStagingStatements(): string[] {
   return [...TRANSIENT_STAGING_TABLES, ...LEGACY_TRANSIENT_STAGING_TABLES]
     .reverse()