diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 2bdfdf2e..e15e693d 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -222,7 +222,11 @@ jobs: # EXISTS). These are pure additions with safe defaults (INTEGER NOT NULL DEFAULT 0 / nullable TEXT), so # no backfill or completion marker is needed — an ALTER populates every existing row. Malformed # responses are fatal. This also makes 0006/0007 replay-safe when they were applied out-of-ledger. - - name: Apply amendment restated/suspect columns + # #306 folds in here rather than adding a second step of its own (as PR #308's own note asked): + # refresh-slice.sql and promote-amendments.sql write contract_number_raw + link_method into served + # `amendments`, so the first cron after release would crash on the missing columns. One probe, one + # mechanism — a second hand-written step is another chance for the alias bug #310 had to fix. + - name: Apply amendment restated/suspect + provenance columns if: steps.guard.outputs.ok == 'true' run: | node scripts/wrangler-render.mjs apps/web/wrangler.jsonc @@ -235,7 +239,9 @@ jobs: --command "SELECT (SELECT COUNT(*) FROM pragma_table_info('amendments') WHERE name = 'value_restated') AS value_restated, (SELECT COUNT(*) FROM pragma_table_info('amendments') WHERE name = 'value_treatment') AS value_treatment, - (SELECT COUNT(*) FROM pragma_table_info('amendments') WHERE name = 'value_suspect') AS value_suspect")" + (SELECT COUNT(*) FROM pragma_table_info('amendments') WHERE name = 'value_suspect') AS value_suspect, + (SELECT COUNT(*) FROM pragma_table_info('amendments') WHERE name = 'contract_number_raw') AS contract_number_raw, + (SELECT COUNT(*) FROM pragma_table_info('amendments') WHERE name = 'link_method') AS link_method")" read_flag() { printf '%s' "$schema_json" | node -e ' @@ -276,6 +282,9 @@ jobs: ensure_column value_restated "value_restated INTEGER NOT NULL DEFAULT 0" ensure_column value_treatment "value_treatment TEXT" ensure_column value_suspect "value_suspect INTEGER NOT NULL DEFAULT 0" + # #306 provenance: NULL on both = the row linked by contract_number directly (or is unlinked). + ensure_column contract_number_raw "contract_number_raw TEXT" + ensure_column link_method "link_method TEXT" # `run deploy`, not `deploy` — bare `pnpm deploy` is a pnpm built-in, not our package script. - name: Deploy explorer (sigma) diff --git a/docs/README.md b/docs/README.md index 01cda08a..16f03f97 100644 --- a/docs/README.md +++ b/docs/README.md @@ -11,7 +11,11 @@ - [`etl-architecture.md`](etl-architecture.md) — целевата ETL архитектура (RFC): предложение за състоянието и реда на изпълнение. - [`v1-implementation-plan.md`](v1-implementation-plan.md) — precompute слоят и пагинацията (защо rollup-и и keyset вместо per-request GROUP BY / OFFSET). - [`implementation-plans/286-ocds-amendment-unp.md`](implementation-plans/286-ocds-amendment-unp.md) — защо OCDS анексите не се свързват с договор (OCID вместо УНП) и планът за поправка през bridge-а `tender.id → УНП` + prefer-EOP dedup (#286). +<<<<<<< HEAD +- [`implementation-plans/306-amendment-contract-namespace-link.md`](implementation-plans/306-amendment-contract-namespace-link.md) — защо 1 937 EOP анекса не се свързват с договор (номерът на анекса е в друго именно пространство от деловодния номер) и планът за поправка чрез value-anchor (`value_before → signing_value`, 99.99% точност) (#306). +======= - [`implementation-plans/305-amendment-value-double-count.md`](implementation-plans/305-amendment-value-double-count.md) — защо стойността на анекс се удвоява (ЦАИС ЕОП слага новия **тотал** в полето за промяна) и планът за откриване/поправка: `annex_total_suspect` флаг + текстова хеуристика за възстановяване на истинския тотал (#305). +>>>>>>> origin/main - [`integrity-gate.md`](integrity-gate.md) — reconciliation gate-ът: hard asserts върху тоталите при import/CI. - [`anomaly-report.md`](anomaly-report.md) — cross-row аномалии при опресняване: какво `value_flag` не хваща на ниво отделен договор. - [`deploy.md`](deploy.md) — деплой към Cloudflare: двата Worker-а (`sigma`, `sigma-etl`) и споделеният D1 per environment. diff --git a/docs/implementation-plans/306-amendment-contract-namespace-link.md b/docs/implementation-plans/306-amendment-contract-namespace-link.md new file mode 100644 index 00000000..3b23ff19 --- /dev/null +++ b/docs/implementation-plans/306-amendment-contract-namespace-link.md @@ -0,0 +1,94 @@ +# Implementation Plan: #306 — annexes don't link to a contract (contract number in a different namespace) + +## Executive Summary + +| Field | Value | +|---|---| +| Ticket | [midt-bg/sigma#306](https://github.com/midt-bg/sigma/issues/306) — labels `data-quality`, `etl`, `priority: high` | +| Problem | 1,937 EOP annexes (7.2%) don't link to any contract. #286 fixed the OCDS half (procedure axis: OCID vs УНП). This is the other axis: the **contract number**. The annex carries an internal annex-side number (`148846`, `2886`); the contract carries the buyer's filing number (`Д-226`, `388-2020`). The exact `(unp, contract_number)` join fails, so the annex drops out of every annex→contract→company/authority rollup. | +| Root cause | Two genuinely unrelated identifiers in different namespaces — not dirty strings. Measured: `TRIM`+`UPPER` saves 0, digits-only 11, substring 19. String normalisation is a dead end. | +| Approach | Link by **value**: an annex's `value_before` is the contract's value at amendment time, so it equals the target contract's `signing_value`. Rewrite `raw_amendments.contract_number` to the resolved contract when `value_before` **exactly** (< 0.5 стотинка), currency-matched, uniquely matches one contract on the procedure; propagate across a chain sharing the annex-side number. | +| Complexity | Medium — the resolver is a self-contained CTE; the subtlety is full-vs-slice parity (the #305 bug class) and the slice's windowed staging. | +| Risk | Low on the full path (measured 99.99% precision), gated to leave ambiguous/no-match annexes unlinked. | +| Status | **Full-derive resolver implemented as a gated, standalone script (`scripts/resolve-amendment-contracts.sql`) that runs BEFORE `derive-amendments.sql`, full-derive path only. Slice/daily resolver deferred (documented below).** Validated against the live `sigma-dev` corpus; hardened per the PR #308 review (ordering blocker, group-contradiction rule, cumulative-candidate dedup, EIK guard, provenance columns). | + +--- + +## 1. Problem, verified on real data (`sigma-dev`) + +Reproduced the issue's breakdown exactly on the served corpus (an amendment `a` links iff `contracts c` exists with `c.tender_id = 't:'||a.unp AND c.contract_number = a.contract_number`): + +| check | #306 | live `sigma-dev` | +|---|---|---| +| EOP annexes unlinked | 1930 | **1937** | +| empty contract_number | 0 | 0 | +| empty УНП | 0 | 0 | +| УНП not in `tenders` | 0 | 3 | +| procedure has no contract | 12 | 15 | +| procedure has contracts, none match | 1918 | **1919** | +| procedure has exactly 1 contract (safe) | 1012 | **1012** | +| price-changing | 554 | **555** | + +## 2. The value anchor (the insight beyond the issue) + +The issue proposed linking only the 1,012 single-contract procedures by УНП. Measuring `value_before` → `signing_value` unlocks the multi-contract cases too. On the **already-linked** annexes (ground truth), when `value_before` uniquely matches one contract's `signing_value`: + +| gate | unique matches | correct | precision | +|---|---|---|---| +| within 1% | 9061 | 8975 | 99.08% | +| **exact (< 0.5 стотинка)** | **9349** | **9348** | **99.99%** | + +Exact-cent match wins on **both** precision and recall — the 1% band manufactures ambiguity and admits the #305 "matched a smaller sibling's value" errors. Example `00011-2020-0002` (multi-lot): the unlinked `2886` annexes carry `value_before=56000`, which exactly matches contract `388-2020` (lot 2, signing 56000), **not** `387-2020` (64000) — a case УНП-only linking cannot resolve. + +### Recovery under the shipped gate (exact + currency + unique + chain-propagation) + +| tier | rule | links | +|---|---|---| +| single-contract, value-confirmed | procedure has 1 contract, `value_before` == its `signing_value` | 775 | +| multi-contract, value-unique | `value_before` exactly matches one of ≥2 contracts | 604 | +| **combined (before chain propagation)** | | **~1,379 (71%)** | + +Left **unlinked by design**: value-ambiguous (matches 2+, ~115), no-match (~193, target not yet ingested — the #249 class), no-tender/no-contract (~18). An honest gap beats a wrong contract on a transparency site. + +> **Re-measured on the live `sigma-dev` corpus (PR #308 review).** Read-only reproduction against the served corpus confirms the headline numbers hold under the revised rule: **1,937** unlinked, **1,563** recovered (1,377 direct + 186 propagated), precision **9,348/9,349 = 99.99%**. Two review inferences were corrected by the data: the `value_before IS NULL/≤0` class is **empty** (all 1,937 unlinked annexes carry a value; the `1,379+326≠1,937` gap is tier-estimate rounding, not a value-less class), and the only anchor-disagreement group in the whole corpus is the correct 00026 lot-base case (hence the §3.4 rule revision). **Still owed:** the served corpus is deduped one-row-per-contract, so it does not exercise the cumulative-staging path (§3.1) — that must be re-measured on raw cumulative staging via a local full backfill before the dedup fix is fully validated end-to-end. + +## 3. Fix — full-derive path (`scripts/resolve-amendment-contracts.sql`, run before `derive-amendments.sql`) + +The resolver is a **standalone script** that `import.mjs` runs from `runFullDerive` / `runWorkBackfill` **before** `derive-amendments.sql` — i.e. **before** the #286 prefer-EOP dedup DELETE and its diagnostics. Ordering is load-bearing (PR #308 review, todorkolev #1 blocker): rewriting an EOP annex onto a contract that already kept an OCDS twin, if done *after* the dedup, resurrects the twin (`annex_count = 2` on a one-annex contract) and trips the `amendment-twin-dedup` integrity gate (#303), failing the whole derive. Running first — and above the #286 diagnostics — keeps the dedup the sole twin guard and its dropped/excess counts honest bounds. The resolver reads only `source LIKE 'eop:%'` rows + `raw_contracts`, so it is independent of the OCDS bridge and safe to run first. + +1. `contract_candidates` — `raw_contracts` **deduped to one row per `(unp, contract_number)`** (`ROW_NUMBER() … ORDER BY source DESC, id DESC = 1`), mirroring normalize-raw. `raw_contracts` is cumulative (the same contract recurs across daily buckets; the collapse happens later in normalize-raw), so without this `COUNT(*) OVER` would count staging **rows**, not contracts — mass fail-close on a real rebuild, or a match to a superseded value (PR #308 review, nikimilenkov HIGH 2). +2. `grp` — **all** EOP annexes with no `(unp, contract_number)` contract, grouped by the shared annex-side number `(unp, annex_cnum)` = one contract's chain. Value-less members (`value_before` NULL/≤0) are included so they can inherit the chain target (review MEDIUM 2). +3. `vmatch` — join to `contract_candidates` on `unp`, exact value (`ABS(signing_value − value_before) < 0.005`), **explicit** currency on both sides (blank ≠ blank; review LOW 1), null-tolerant contractor-EIK match (a value collision onto a different contractor is refused for free; review MEDIUM 5). `COUNT(*) OVER (PARTITION BY amendment_id)` flags uniqueness. +4. **Group rule** (reviews todorkolev #2, nikimilenkov MEDIUM 1 & 2, revised against the live corpus): a member's OWN unique (`n_match = 1`) exact match always applies — it is individually trustworthy (the 99.99% figure) and is **not** voided when annex-number siblings point elsewhere, because an annex number can be a **lot-base** shared across contracts (live corpus: `20РП-У50А015` → …-Л01 @ 22569.98 **and** …-Л03 @ 28557.50, each annex exactly-uniquely matching its own lot — the sole disagreement group in the whole corpus, and it is correct). Members with **no** own unique match (value-less admin steps, or later steps whose cumulative value matches no `signing_value`) inherit one agreed group target — but **only** when the direct members agree; disagreement withholds *propagation*, never the direct hits. A member that is itself value-ambiguous (`n_match ≥ 2`) carries its own contradicting evidence and never links, directly or by inheritance. + + > An earlier revision refused the whole group on disagreement (nikimilenkov MEDIUM 1 as first stated). Measured on the live corpus that dropped 2 confirmed-correct multi-lot links and prevented **zero** wrong links (the only disagreement group is the benign 00026 lot-base case), so the rule keeps direct hits and gates only propagation. +5. `UPDATE raw_amendments SET contract_number = resolved, contract_number_raw = , link_method = 'value_anchor'` in place — like the #286 УНП bridge, but **preserving provenance** (review MEDIUM 4). The rollup, `promote-amendments.sql`, and the serving join then link with no further change; the original annex number stays in `contract_number_raw`, which also keeps it in the amendment `natural_key` so a resolved row never collides with a native annex sharing `document_number` on the target (review MEDIUM 3). + +Diagnostic printed by wrangler: `annexes_value_linked`, `eop_annexes_still_unlinked` (complementary predicates over the same mismatch population). + +Idempotent: after the rewrite the row links by number, so a re-run's `grp` no longer selects it and the provenance columns are never re-stamped. + +## 4. Slice / daily path — **implemented** (corpus-safe resolver inside `refresh-slice.sql`) + +Originally deferred: the standalone `resolve-amendment-contracts.sql` is **not** run on the slice path (`runSliceDerive`), because it would execute against `refresh-slice.sql`'s **windowed** `raw_contracts`, where "unique on the procedure" means "unique **in the window**" — a corpus-ambiguous annex looks unique in a narrow window and mislinks, and the measured 99.99% precision (a full-corpus number) would not carry (PR #308 review, nikimilenkov HIGH 1). + +The daily/slice + Worker path now runs a corpus-safe equivalent **inside `refresh-slice.sql`** (search `#306: slice-safe value-anchor resolver`), addressing PR #308 review todorkolev "дневните обновявания": the cron runs only `refresh-slice.sql`, so the fix had to live there or stay inert in production. It applies the same value/currency/EIK anchor, chain rules, and provenance stamping as the full path, differing only in the candidate source: + +- Candidate contracts are drawn from the **served `contracts` table** (the whole corpus; `unp` via the `tender_id` suffix, contractor ЕИК via the winning `bidders.eik_normalized`) **UNIONed** with this window's `raw_contracts`, so uniqueness is asked corpus-wide. It is intentionally **not** under a byte-identical lockstep marker — the candidate source differs by construction. Scans are bounded to procedures with an EOP annex in this window (`window_unps`), keeping the served read a keyed lookup (`idx_contracts_tender_id`). +- Resolved targets land in `refresh_touched_contracts` **for free**: the resolver rewrites the target `contract_number` onto `raw_amendments` in the setup batch, and the existing `@refresh-batch amendments` touch join (`raw_amendments` → `contracts` on `contract_number`) then scopes prior-window targets into it. +- It runs in the setup batch **before** the prefer-EOP dedup DELETE, the same ordering the full path uses before `derive-amendments.sql`'s dedup (review todorkolev #1 blocker — otherwise a rewrite resurrects an OCDS twin and trips `amendment-twin-dedup` #303). +- The "is this annex's number already a real contract?" question is asked over a **value-agnostic** `all_contract_numbers` CTE (every contract number on the procedure, from window `raw_contracts` **and** served `contracts`, regardless of `signing_value`) — **not** over `contract_candidates` (which requires `signing_value > 0` for value matching). This keeps the slice path identical to the full path's `NOT EXISTS … raw_contracts`: an annex that matches a **zero-value** contract by number links by number, and is never value-linked to a neighbour (review todorkolev discrepancy). + +**Deployment (review todorkolev blocker).** `refresh-slice.sql` now writes `contract_number_raw` + `link_method` into served `amendments`, but the deployed DBs are populated out-of-band so `wrangler d1 migrations apply` can't run. `deploy.yml` gains an idempotent step (probe `pragma_table_info`, `ALTER` only the missing columns, malformed response fatal) **before** the Worker deploys — otherwise the first cron after release crashes on the missing columns. If #307 merges first, fold these two columns into its provenance step instead. + +Provenance keeps slice and full keys aligned: the amendment `natural_key` (winners dedup + promotion) uses the raw annex number via `COALESCE(NULLIF(contract_number_raw,''), contract_number, '')`, and the served `amendments` promotion carries `contract_number_raw` + `link_method`. A resolved annex re-emitted in a later slice window re-resolves deterministically to the same key — an honest, never-double-counted link. Where a target contract and its namespace-mismatched annex arrive in the **same** window, the annex still links (window `raw_contracts` is in the candidate union); the residual best-effort gap is only a target never served or staged, which the next full rebuild closes (#286 precedent). + +## 5. Blast radius (auto-corrects on a full rebuild) + +Everything is rebuilt from scratch by `precompute.sql` (DELETE/INSERT), so it self-corrects: `company_totals`, `authority_totals`, `sector_totals`, `home_totals`, `facet_counts`, `flow_pairs`, `cpv_division_stats`, `search_index`, and each contract's `annex_count`/`current_value`/timeline. Overall annex counts and sums on the annexes themselves are unchanged. Two things to verify after the run: (1) `cpv_division_stats` p95/p99 bands shift for affected divisions; (2) **merge-order dependency** — ~23 newly-linked annexes have `value_after ≥ 2× value_before` and should trip the #307 double-count flags (`annex_total_suspect`/`value_suspect`). Those flags do **not** exist on this branch (they are #307, in review); if this merges before #307, the newly-linked ≥2× annexes enter the aggregates **unflagged** until #307 lands. Track the merge order explicitly. + +**Migration numbering (PR #308 review todorkolev "сблъсък на номера").** #307 adds `0006_amendment_restated.sql` + `0007_amendment_value_suspect.sql`; this branch's provenance migration is therefore numbered **`0008_amendment_provenance.sql`** to sit after both. This assumes #307 merges first (it should — the ≥2× flags above depend on it). If #308 lands first instead, renumber to `0006` and have #307 shift to `0007`/`0008`. + +## 6. Tests + +`packages/db/src/amendments-contract-resolve.test.ts` runs the real `resolve-amendment-contracts.sql` + `derive-amendments.sql` (the full-derive composition) against SQLite and pins: single-contract link (00017 shape), multi-lot value disambiguation (00011 shape), chain propagation + `current_value` = last step, value-less chain member inherits (MEDIUM 2), ambiguous-in-chain member stays unlinked (todorkolev #2), whole-group refusal on disagreeing anchors (MEDIUM 1), cumulative-staging duplicate counted once (HIGH 2), EIK guard (MEDIUM 5), value-ambiguous / no-match / currency-guard / blank-vs-blank-currency (LOW 1) → unlinked, twin-ordering (the resolver runs before the prefer-EOP dedup, no `annex_count = 2` — todorkolev #1 blocker), gate (derive-alone does not resolve — HIGH 1), idempotency, natural-key collision avoidance (MEDIUM 3), provenance through promote into served `amendments` (MEDIUM 4), already-linked untouched, and the diagnostic counts. diff --git a/packages/db/migrations/0008_amendment_provenance.sql b/packages/db/migrations/0008_amendment_provenance.sql new file mode 100644 index 00000000..fc8d4727 --- /dev/null +++ b/packages/db/migrations/0008_amendment_provenance.sql @@ -0,0 +1,9 @@ +-- #306: provenance for value-anchor-linked annexes. The resolver in +-- scripts/resolve-amendment-contracts.sql rewrites a namespace-mismatched annex's contract_number to its +-- target contract; unlike the #286 OCDS bridge (whose OCID survives in tender_ext_id + source), the annex +-- number would otherwise be destroyed with no trace, making the 99.99%-precision claim unauditable in +-- production and any "this annex isn't ours" complaint uninvestigable (review nikimilenkov MEDIUM 4). +-- Keep the original annex-side number and stamp the link method so value-linked rows stay enumerable on the +-- served side. NULL on both columns = the row linked by contract_number directly (or is unlinked). +ALTER TABLE amendments ADD COLUMN contract_number_raw TEXT; +ALTER TABLE amendments ADD COLUMN link_method TEXT; diff --git a/packages/db/src/amendments-contract-resolve.test.ts b/packages/db/src/amendments-contract-resolve.test.ts new file mode 100644 index 00000000..f0b57d13 --- /dev/null +++ b/packages/db/src/amendments-contract-resolve.test.ts @@ -0,0 +1,634 @@ +// Issue #306 — EOP annexes whose annex-side number is in a different namespace than the contract number +// (the annex carries an internal number like 148846; the contract carries the buyer's filing number like +// Д-226), so the (unp, contract_number) join drops them out of every annex→contract rollup. The resolver in +// scripts/resolve-amendment-contracts.sql links them by the exact, currency- and contractor-matched +// value_before → signing_value anchor (measured 99.99% precision on the real corpus), uniquely, with chain +// propagation, leaving ambiguous / no-match annexes honestly unlinked. +// +// The resolver is a SEPARATE script that runs BEFORE derive-amendments.sql on the full-derive path only +// (review todorkolev #1 blocker: running after the prefer-EOP dedup resurrects OCDS twins; review +// nikimilenkov HIGH 1: the slice path's windowed raw_contracts can't answer "unique on the procedure"). +// These tests run the REAL scripts against SQLite via the sqlite3 CLI, exactly as runFullDerive composes +// them — resolver, then derive — so the value anchor, dedup, group rule, guards, and provenance are +// exercised as shipped. +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); +const initSchema = resolve(root, 'packages/db/migrations/0000_init.sql'); +// #305: promote-amendments.sql writes value_restated/value_treatment/value_suspect into served amendments. +const restatedMigration = resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'); +const valueSuspectMigration = resolve( + root, + 'packages/db/migrations/0007_amendment_value_suspect.sql', +); +const provenanceMigration = resolve(root, 'packages/db/migrations/0008_amendment_provenance.sql'); +const workStagingSchema = resolve(root, 'scripts/work-staging-schema.sql'); +const resolveAmendments = resolve(root, 'scripts/resolve-amendment-contracts.sql'); +const deriveAmendments = resolve(root, 'scripts/derive-amendments.sql'); +const promoteAmendments = resolve(root, 'scripts/promote-amendments.sql'); + +function sqlite(dbPath: string, sql: string): void { + execFileSync('sqlite3', [dbPath], { input: sql, encoding: 'utf8', stdio: 'pipe' }); +} + +function sqliteJson(dbPath: string, sql: string): T[] { + const out = execFileSync('sqlite3', ['-json', dbPath, sql], { encoding: 'utf8' }).trim(); + return out ? (JSON.parse(out) as T[]) : []; +} + +function readScript(dbPath: string, path: string): void { + execFileSync('sqlite3', [dbPath], { + input: `PRAGMA foreign_keys=ON;\n.read ${path}\n`, + stdio: 'pipe', + }); +} + +function readScriptCapture(dbPath: string, path: string): string { + return execFileSync('sqlite3', [dbPath], { + input: `PRAGMA foreign_keys=ON;\n.read ${path}\n`, + encoding: 'utf8', + }); +} + +// The full-derive composition: value resolver first (review todorkolev #1), then the derive rollup. +function runFullDerive(dbPath: string): void { + readScript(dbPath, resolveAmendments); + readScript(dbPath, deriveAmendments); +} + +// The numeric rows the resolver prints; the #306 diagnostic is its only 2-column row +// (annexes_value_linked | eop_annexes_still_unlinked). +function diagRows(out: string): number[][] { + return out + .split('\n') + .map((line) => line.trim()) + .filter((line) => /^\d+(\|\d+)*$/.test(line)) + .map((line) => line.split('|').map(Number)); +} + +let dir: string; +let db: string; + +beforeEach(() => { + dir = mkdtempSync(resolve(tmpdir(), 'amendments-resolve-')); + db = resolve(dir, 'work.sqlite'); + readScript(db, initSchema); + readScript(db, restatedMigration); + readScript(db, valueSuspectMigration); + readScript(db, provenanceMigration); + readScript(db, workStagingSchema); +}); + +afterEach(() => { + rmSync(dir, { recursive: true, force: true }); +}); + +describe('#306 amendment→contract value-anchor resolver', () => { + it('links a single-contract procedure whose annex number differs (00017 shape) by exact value', () => { + // UNP with ONE contract "ОП-3-016/…" signing 5800; annex carries internal number 11725, value_before + // 5800 (exact). The number join fails; the value anchor links it. + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00017-2020-0041','ОП-3-016/03.02.2021г.',5800,'BGN'); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, value_delta, currency) VALUES + ('eop:annexes:2026-03-05','2026-03-05','00017-2020-0041','11725','2026-03-05','A1',5800,5800,0,'BGN');`, + ); + runFullDerive(db); + + // The annex's contract_number is rewritten to the real filing number → it now links. + expect( + sqliteJson<{ contract_number: string; contract_number_raw: string; link_method: string }>( + db, + "SELECT contract_number, contract_number_raw, link_method FROM raw_amendments WHERE document_number='A1'", + ), + ).toEqual([ + { + contract_number: 'ОП-3-016/03.02.2021г.', + contract_number_raw: '11725', + link_method: 'value_anchor', + }, + ]); + // The contract picks up the annex in its rollup. + expect( + sqliteJson<{ annex_count: number }>( + db, + "SELECT annex_count FROM raw_contracts WHERE contract_number='ОП-3-016/03.02.2021г.'", + ), + ).toEqual([{ annex_count: 1 }]); + }); + + it('disambiguates a multi-contract (multi-lot) procedure by value (00011 shape)', () => { + // Two contracts on one procedure: 387-2020 (signing 64000) and 388-2020 (signing 56000). An unlinked + // annex carries internal number 2886, value_before 56000 → must link to 388-2020, NEVER 387-2020. + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00011-2020-0002','387-2020',64000,'BGN'), + ('eop:contracts:2026-03-05','2026-03-05','00011-2020-0002','388-2020',56000,'BGN'); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, value_delta, currency) VALUES + ('eop:annexes:2026-03-05','2026-03-05','00011-2020-0002','2886','2026-03-05','A1',56000,55552.5,NULL,'BGN');`, + ); + runFullDerive(db); + + expect( + sqliteJson<{ contract_number: string }>( + db, + "SELECT contract_number FROM raw_amendments WHERE document_number='A1'", + ), + ).toEqual([{ contract_number: '388-2020' }]); + expect( + sqliteJson<{ contract_number: string; annex_count: number }>( + db, + "SELECT contract_number, annex_count FROM raw_contracts WHERE unp='00011-2020-0002' ORDER BY contract_number", + ), + ).toEqual([ + { contract_number: '387-2020', annex_count: 0 }, + { contract_number: '388-2020', annex_count: 1 }, + ]); + }); + + it('propagates the resolved target across a chain sharing the annex number, so current_value is the last step', () => { + // Three annexes share internal number 2886; only the FIRST carries value_before = signing (56000), the + // later two carry the running cumulative. All must attach to 388-2020, and current_value must be the + // LAST step's value_after (53580.21), not the first. + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00011-2020-0002','387-2020',64000,'BGN'), + ('eop:contracts:2026-03-05','2026-03-05','00011-2020-0002','388-2020',56000,'BGN'); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('eop:annexes:2026-03-05','2026-03-05','00011-2020-0002','2886','2026-03-05','A1',56000,55552.5,'BGN'), + ('eop:annexes:2026-03-05','2026-03-06','00011-2020-0002','2886','2026-03-06','A2',55552.5,55306.05,'BGN'), + ('eop:annexes:2026-03-05','2026-03-07','00011-2020-0002','2886','2026-03-07','A3',55306.05,53580.21,'BGN');`, + ); + runFullDerive(db); + + // All three annexes now carry the resolved contract_number. + expect( + sqliteJson<{ n: number }>( + db, + "SELECT COUNT(*) AS n FROM raw_amendments WHERE unp='00011-2020-0002' AND contract_number='388-2020'", + ), + ).toEqual([{ n: 3 }]); + // annex_count = 3, current_value = the latest step's after-value. + expect( + sqliteJson<{ annex_count: number; current_value: number }>( + db, + "SELECT annex_count, current_value FROM raw_contracts WHERE contract_number='388-2020'", + ), + ).toEqual([{ annex_count: 3, current_value: 53580.21 }]); + }); + + it('propagates the target to a VALUE-LESS chain member so the chain does not break (review MEDIUM 2)', () => { + // A1 carries value_before = signing (unique anchor → 388-2020); A2 is an administrative annex with NO + // value_before (a term/scope change). A2 shares the annex number 2886, so it must INHERIT 388-2020 — + // else the chain breaks and annex_count under-counts. + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00011-2020-0002','387-2020',64000,'BGN'), + ('eop:contracts:2026-03-05','2026-03-05','00011-2020-0002','388-2020',56000,'BGN'); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('eop:annexes:2026-03-05','2026-03-05','00011-2020-0002','2886','2026-03-05','A1',56000,55552.5,'BGN'), + ('eop:annexes:2026-03-05','2026-03-06','00011-2020-0002','2886','2026-03-06','A2',NULL,NULL,'BGN');`, + ); + runFullDerive(db); + + expect( + sqliteJson<{ document_number: string; contract_number: string; link_method: string | null }>( + db, + 'SELECT document_number, contract_number, link_method FROM raw_amendments ORDER BY document_number', + ), + ).toEqual([ + { document_number: 'A1', contract_number: '388-2020', link_method: 'value_anchor' }, + { document_number: 'A2', contract_number: '388-2020', link_method: 'value_anchor' }, + ]); + expect( + sqliteJson<{ annex_count: number }>( + db, + "SELECT annex_count FROM raw_contracts WHERE contract_number='388-2020'", + ), + ).toEqual([{ annex_count: 2 }]); + }); + + it('does NOT attach an ambiguous chain member to the group target (review todorkolev #2)', () => { + // Procedure: C-1 @ 500, C-2 @ 500, C-3 @ 1000. Annex 999: A1 value_before 1000 → C-3 (unique anchor); + // A2 value_before 500 → matches C-1 AND C-2 (ambiguous). A2 must stay unlinked — it carries its own + // contradicting evidence — and must NOT corrupt C-3's current_value. + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00050-2020-0001','C-1',500,'BGN'), + ('eop:contracts:2026-03-05','2026-03-05','00050-2020-0001','C-2',500,'BGN'), + ('eop:contracts:2026-03-05','2026-03-05','00050-2020-0001','C-3',1000,'BGN'); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('eop:annexes:2026-03-05','2026-03-05','00050-2020-0001','999','2026-03-05','A1',1000,900,'BGN'), + ('eop:annexes:2026-03-05','2026-03-06','00050-2020-0001','999','2026-03-06','A2',500,450,'BGN');`, + ); + runFullDerive(db); + + expect( + sqliteJson<{ document_number: string; contract_number: string }>( + db, + 'SELECT document_number, contract_number FROM raw_amendments ORDER BY document_number', + ), + ).toEqual([ + { document_number: 'A1', contract_number: 'C-3' }, // unique anchor linked + { document_number: 'A2', contract_number: '999' }, // ambiguous → stays unlinked + ]); + // C-3 keeps A1 only; current_value is A1's after-value (900), NOT the ambiguous A2's 450. + expect( + sqliteJson<{ annex_count: number; current_value: number }>( + db, + "SELECT annex_count, current_value FROM raw_contracts WHERE contract_number='C-3'", + ), + ).toEqual([{ annex_count: 1, current_value: 900 }]); + }); + + it('links each annex of a lot-base group to its OWN lot by its own unique match (real 00026 shape)', () => { + // The annex-side number is a LOT-BASE shared across two lots (real corpus: 20РП-У50А015 → …-Л01 @ + // 22569.98 AND …-Л03 @ 28557.50). Each annex exactly-uniquely matches its OWN lot, so BOTH link — the + // "disagreement" between siblings is benign and must NOT void the individually-trustworthy direct hits. + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00026-2020-0027','20РП-У50А015-Л01',22569.98,'BGN'), + ('eop:contracts:2026-03-05','2026-03-05','00026-2020-0027','20РП-У50А015-Л03',28557.5,'BGN'); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('eop:annexes:2026-03-05','2026-03-05','00026-2020-0027','20РП-У50А015','2026-03-05','A1',22569.98,22000,'BGN'), + ('eop:annexes:2026-03-05','2026-03-06','00026-2020-0027','20РП-У50А015','2026-03-06','A2',28557.5,28000,'BGN');`, + ); + runFullDerive(db); + + expect( + sqliteJson<{ document_number: string; contract_number: string }>( + db, + "SELECT document_number, contract_number FROM raw_amendments WHERE unp='00026-2020-0027' ORDER BY document_number", + ), + ).toEqual([ + { document_number: 'A1', contract_number: '20РП-У50А015-Л01' }, + { document_number: 'A2', contract_number: '20РП-У50А015-Л03' }, + ]); + expect( + sqliteJson<{ contract_number: string; annex_count: number }>( + db, + "SELECT contract_number, annex_count FROM raw_contracts WHERE unp='00026-2020-0027' ORDER BY contract_number", + ), + ).toEqual([ + { contract_number: '20РП-У50А015-Л01', annex_count: 1 }, + { contract_number: '20РП-У50А015-Л03', annex_count: 1 }, + ]); + }); + + it('withholds PROPAGATION when the group disagrees, but keeps the direct hits (review MEDIUM 1, revised)', () => { + // Two direct hits disagree (lot-base spread), plus a value-less admin annex A3 sharing the number. The + // two direct hits still link to their own lots; A3 has no agreed target to inherit → stays unlinked. + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00060-2020-0001','D-1',100,'BGN'), + ('eop:contracts:2026-03-05','2026-03-05','00060-2020-0001','D-2',200,'BGN'); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('eop:annexes:2026-03-05','2026-03-05','00060-2020-0001','777','2026-03-05','A1',100,90,'BGN'), + ('eop:annexes:2026-03-05','2026-03-06','00060-2020-0001','777','2026-03-06','A2',200,180,'BGN'), + ('eop:annexes:2026-03-05','2026-03-07','00060-2020-0001','777','2026-03-07','A3',NULL,NULL,'BGN');`, + ); + runFullDerive(db); + + expect( + sqliteJson<{ document_number: string; contract_number: string }>( + db, + "SELECT document_number, contract_number FROM raw_amendments WHERE unp='00060-2020-0001' ORDER BY document_number", + ), + ).toEqual([ + { document_number: 'A1', contract_number: 'D-1' }, // own unique match stands + { document_number: 'A2', contract_number: 'D-2' }, // own unique match stands + { document_number: 'A3', contract_number: '777' }, // no agreed target → no propagation + ]); + expect( + sqliteJson<{ contract_number: string; annex_count: number }>( + db, + "SELECT contract_number, annex_count FROM raw_contracts WHERE unp='00060-2020-0001' ORDER BY contract_number", + ), + ).toEqual([ + { contract_number: 'D-1', annex_count: 1 }, + { contract_number: 'D-2', annex_count: 1 }, + ]); + }); + + it('counts cumulative-staging duplicates of one contract as a SINGLE candidate (review HIGH 2)', () => { + // The same logical contract is staged from TWO daily EOP buckets (raw_contracts is cumulative). Without + // deduping candidates, the lone annex would see n_match = 2 and be refused; the resolver must collapse + // the duplicate to one candidate and link. + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00070-2020-0001','FILING-7',12345,'BGN'), + ('eop:contracts:2026-03-06','2026-03-06','00070-2020-0001','FILING-7',12345,'BGN'); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('eop:annexes:2026-03-06','2026-03-06','00070-2020-0001','88001','2026-03-06','A1',12345,12000,'BGN');`, + ); + runFullDerive(db); + + expect( + sqliteJson<{ contract_number: string }>( + db, + "SELECT contract_number FROM raw_amendments WHERE document_number='A1'", + ), + ).toEqual([{ contract_number: 'FILING-7' }]); + }); + + it('applies an EIK guard — a value match onto a different contractor is refused (review MEDIUM 5)', () => { + // Same unp, value, currency, but the annex is contractor 222222222 and the only value-matching contract + // is contractor 111111111 → the row's own EIK contradicts the target → no link. + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency, contractor_eik) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00080-2020-0001','FILING-8',9000,'BGN','111111111'); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency, contractor_eik) VALUES + ('eop:annexes:2026-03-05','2026-03-05','00080-2020-0001','44001','2026-03-05','A1',9000,8500,'BGN','222222222');`, + ); + runFullDerive(db); + + expect( + sqliteJson<{ contract_number: string }>( + db, + "SELECT contract_number FROM raw_amendments WHERE document_number='A1'", + ), + ).toEqual([{ contract_number: '44001' }]); // unchanged → EIK guard held + }); + + it('leaves a value-AMBIGUOUS annex unlinked (two contracts share the signing value)', () => { + // Both contracts on the procedure have signing 5000; the annex value_before 5000 matches BOTH — the + // resolver must refuse (an honest gap beats a coin-flip). + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00099-2020-0001','LOT-A',5000,'BGN'), + ('eop:contracts:2026-03-05','2026-03-05','00099-2020-0001','LOT-B',5000,'BGN'); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('eop:annexes:2026-03-05','2026-03-05','00099-2020-0001','7777','2026-03-05','A1',5000,4800,'BGN');`, + ); + runFullDerive(db); + + expect( + sqliteJson<{ contract_number: string }>( + db, + "SELECT contract_number FROM raw_amendments WHERE document_number='A1'", + ), + ).toEqual([{ contract_number: '7777' }]); // unchanged → stays unlinked + expect( + sqliteJson<{ total: number }>( + db, + "SELECT COALESCE(SUM(annex_count),0) AS total FROM raw_contracts WHERE unp='00099-2020-0001'", + ), + ).toEqual([{ total: 0 }]); + }); + + it('leaves a NO-MATCH annex unlinked (value_before matches no contract on the procedure)', () => { + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00099-2020-0002','30',700000,'BGN'); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('eop:annexes:2026-03-05','2026-03-05','00099-2020-0002','24035','2026-03-05','A1',123456,120000,'BGN');`, + ); + runFullDerive(db); + + expect( + sqliteJson<{ contract_number: string }>( + db, + "SELECT contract_number FROM raw_amendments WHERE document_number='A1'", + ), + ).toEqual([{ contract_number: '24035' }]); // unchanged + }); + + it('holds the exact-cent tolerance — a value 3 BGN off does NOT match (pins the 0.005 constant)', () => { + // signing 5000, value_before 5003 → 3.00 apart: far under a naive "within 5" band but far over the + // 0.5-стотинка exact gate. Must stay unlinked; this is the whole basis of the 99.99% precision claim. + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00087-2020-0001','FILING-87',5000,'BGN'); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('eop:annexes:2026-03-05','2026-03-05','00087-2020-0001','47001','2026-03-05','A1',5003,4800,'BGN');`, + ); + runFullDerive(db); + + expect( + sqliteJson<{ contract_number: string }>( + db, + "SELECT contract_number FROM raw_amendments WHERE document_number='A1'", + ), + ).toEqual([{ contract_number: '47001' }]); // unchanged → 3 BGN is not an exact-cent match + }); + + it('applies a currency guard — a BGN-valued annex does not match a same-number EUR contract', () => { + // signing_value numerically equals value_before but the currencies differ → not a real value match. + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00207-2020-0171','FILING-1',66820,'EUR'); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('eop:annexes:2026-03-05','2026-03-05','00207-2020-0171','9001','2026-03-05','A1',66820,66000,'BGN');`, + ); + runFullDerive(db); + + expect( + sqliteJson<{ contract_number: string }>( + db, + "SELECT contract_number FROM raw_amendments WHERE document_number='A1'", + ), + ).toEqual([{ contract_number: '9001' }]); // unchanged → currency guard held + }); + + it('does not match blank-currency against blank-currency (review LOW 1)', () => { + // Both sides carry no explicit currency. Under so tight a gate the resolver must NOT silently agree via + // a 'BGN' default — an explicit currency is required on both sides. + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00085-2020-0001','FILING-85',7500,''); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('eop:annexes:2026-03-05','2026-03-05','00085-2020-0001','45001','2026-03-05','A1',7500,7000,'');`, + ); + runFullDerive(db); + + expect( + sqliteJson<{ contract_number: string }>( + db, + "SELECT contract_number FROM raw_amendments WHERE document_number='A1'", + ), + ).toEqual([{ contract_number: '45001' }]); // unchanged → no blank-vs-blank match + }); + + it('does NOT resurrect an OCDS twin: resolver runs before the prefer-EOP dedup (review todorkolev #1 blocker)', () => { + // Contract Д-226 already has an OCDS annex on it. An EOP annex carries the unlinked internal number + // 148846 and value_before = signing (5000). Because the resolver runs BEFORE the #286 prefer-EOP DELETE, + // the EOP annex is moved onto Д-226 first, the OCDS twin is then dropped, and annex_count = 1. If the + // order were reversed the twin would survive and annex_count would be 2 (the twin-dedup #303 gate would + // then hard-fail the whole derive). + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00090-2020-0001','Д-226',5000,'BGN'); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('ocds:annexes:2026-03-05','2026-03-05','00090-2020-0001','Д-226','2026-03-05','ocds-e82gsb-1',5000,NULL,'BGN'), + ('eop:annexes:2026-03-05','2026-03-05','00090-2020-0001','148846','2026-03-05','148846-1',5000,4500,'BGN');`, + ); + runFullDerive(db); + + // Exactly one annex survives on Д-226 (the EOP one), and it drives current_value. + expect( + sqliteJson<{ annex_count: number; current_value: number }>( + db, + "SELECT annex_count, current_value FROM raw_contracts WHERE contract_number='Д-226'", + ), + ).toEqual([{ annex_count: 1, current_value: 4500 }]); + // No twin duplication: exactly one row lands on Д-226 for this procedure. + expect( + sqliteJson<{ n: number }>( + db, + "SELECT COUNT(*) AS n FROM raw_amendments WHERE unp='00090-2020-0001' AND contract_number='Д-226'", + ), + ).toEqual([{ n: 1 }]); + }); + + it('is GATED to the resolver script: derive-amendments.sql alone does not resolve (review HIGH 1)', () => { + // The slice path runs derive-amendments.sql WITHOUT the resolver. Running derive alone must leave a + // resolvable annex untouched — proving the link is gated to the full-derive-only resolver script. + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00095-2020-0001','FILING-95',5800,'BGN'); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('eop:annexes:2026-03-05','2026-03-05','00095-2020-0001','11725','2026-03-05','A1',5800,5600,'BGN');`, + ); + readScript(db, deriveAmendments); // derive ONLY — no resolver + + expect( + sqliteJson<{ contract_number: string; link_method: string | null }>( + db, + "SELECT contract_number, link_method FROM raw_amendments WHERE document_number='A1'", + ), + ).toEqual([{ contract_number: '11725', link_method: null }]); // untouched + }); + + it('is idempotent: a second full derive links nothing new and keeps annex_count stable', () => { + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00096-2020-0001','FILING-96',5800,'BGN'); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('eop:annexes:2026-03-05','2026-03-05','00096-2020-0001','11725','2026-03-05','A1',5800,5600,'BGN');`, + ); + runFullDerive(db); + const secondPass = readScriptCapture(db, resolveAmendments); + readScript(db, deriveAmendments); + + // Second pass over already-resolved staging: the row now links by number so grp is empty and no new + // rewrite happens. The diagnostic still reports the one value-linked row (link_method persists) and 0 + // still unlinked — i.e. nothing new linked, nothing left over. + const twoCol = diagRows(secondPass).filter((r) => r.length === 2); + expect(twoCol[twoCol.length - 1]).toEqual([1, 0]); + // contract_number stays resolved; annex_count stable at 1; provenance preserved. + expect( + sqliteJson<{ annex_count: number }>( + db, + "SELECT annex_count FROM raw_contracts WHERE contract_number='FILING-96'", + ), + ).toEqual([{ annex_count: 1 }]); + expect( + sqliteJson<{ contract_number_raw: string; link_method: string }>( + db, + "SELECT contract_number_raw, link_method FROM raw_amendments WHERE document_number='A1'", + ), + ).toEqual([{ contract_number_raw: '11725', link_method: 'value_anchor' }]); + }); + + it('carries provenance through promote into the served amendments table (review MEDIUM 4)', () => { + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00097-2020-0001','FILING-97',5800,'BGN'); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('eop:annexes:2026-03-05','2026-03-05','00097-2020-0001','11725','2026-03-05','A1',5800,5600,'BGN');`, + ); + runFullDerive(db); + readScript(db, promoteAmendments); + + // The served row is enumerable as value-linked, and the original annex number survives as provenance. + expect( + sqliteJson<{ contract_number: string; contract_number_raw: string; link_method: string }>( + db, + "SELECT contract_number, contract_number_raw, link_method FROM amendments WHERE unp='00097-2020-0001'", + ), + ).toEqual([ + { contract_number: 'FILING-97', contract_number_raw: '11725', link_method: 'value_anchor' }, + ]); + // The audit count nikimilenkov asked for is a plain query on the served side. + expect( + sqliteJson<{ n: number }>( + db, + "SELECT COUNT(*) AS n FROM amendments WHERE link_method='value_anchor'", + ), + ).toEqual([{ n: 1 }]); + }); + + it('a resolved annex does not collide with a native annex sharing document_number on the target (review MEDIUM 3)', () => { + // Contract FILING-98 already has a native annex whose document_number is 'DOC-1'. An unlinked annex 55501 + // resolves to FILING-98 and ALSO has document_number 'DOC-1'. Keying the resolved row on its original + // annex number keeps both rows distinct — the real annex is not silently dropped by the dedup. + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00098-2020-0001','FILING-98',5000,'BGN'); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('eop:annexes:2026-03-05','2026-03-05','00098-2020-0001','FILING-98','2026-03-04','DOC-1',6000,5500,'BGN'), + ('eop:annexes:2026-03-05','2026-03-05','00098-2020-0001','55501','2026-03-05','DOC-1',5000,4500,'BGN');`, + ); + runFullDerive(db); + + // Both annexes survive on FILING-98 (the native one and the value-linked one). + expect( + sqliteJson<{ annex_count: number }>( + db, + "SELECT annex_count FROM raw_contracts WHERE contract_number='FILING-98'", + ), + ).toEqual([{ annex_count: 2 }]); + }); + + it('never touches an annex that already links by contract_number, and emits the #306 diagnostic', () => { + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00001-2020-0001','Д-100',5000,'BGN'), -- annex links by value (unlinked number) + ('eop:contracts:2026-03-05','2026-03-05','00002-2020-0001','Д-200',9000,'BGN'); -- annex already links by number + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('eop:annexes:2026-03-05','2026-03-05','00001-2020-0001','55501','2026-03-05','A1',5000,4500,'BGN'), + ('eop:annexes:2026-03-05','2026-03-05','00002-2020-0001','Д-200','2026-03-05','A2',9000,8000,'BGN');`, + ); + const out = readScriptCapture(db, resolveAmendments); + readScript(db, deriveAmendments); + + // A1 got resolved; A2 (already linked by number) is untouched and keeps NULL provenance. + expect( + sqliteJson<{ document_number: string; contract_number: string; link_method: string | null }>( + db, + 'SELECT document_number, contract_number, link_method FROM raw_amendments ORDER BY document_number', + ), + ).toEqual([ + { document_number: 'A1', contract_number: 'Д-100', link_method: 'value_anchor' }, + { document_number: 'A2', contract_number: 'Д-200', link_method: null }, + ]); + // Diagnostic: exactly 1 value-linked, 0 still unlinked. The resolver's only 2-column row. + const twoCol = diagRows(out).filter((r) => r.length === 2); + expect(twoCol[twoCol.length - 1]).toEqual([1, 0]); + }); +}); diff --git a/packages/db/src/amendments-ocds-link.test.ts b/packages/db/src/amendments-ocds-link.test.ts index 73dd62a5..e86eb5bb 100644 --- a/packages/db/src/amendments-ocds-link.test.ts +++ b/packages/db/src/amendments-ocds-link.test.ts @@ -16,6 +16,8 @@ const initSchema = resolve(root, 'packages/db/migrations/0000_init.sql'); // #305 Tier-2: promote-amendments.sql writes value_restated/value_treatment to served amendments. const migration6 = resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'); const migration7 = resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'); +// #306 provenance columns on served `amendments` — promote/refresh-slice write contract_number_raw + link_method. +const migration8 = resolve(root, 'packages/db/migrations/0008_amendment_provenance.sql'); const workStagingSchema = resolve(root, 'scripts/work-staging-schema.sql'); const deriveAmendments = resolve(root, 'scripts/derive-amendments.sql'); const promoteAmendments = resolve(root, 'scripts/promote-amendments.sql'); @@ -77,6 +79,7 @@ beforeEach(() => { readScript(db, initSchema); // served `amendments` + `contracts` readScript(db, migration6); // #305 Tier-2 value_restated/value_treatment on served amendments readScript(db, migration7); // #305 residual value_suspect on served amendments + readScript(db, migration8); // #306 provenance columns on served amendments readScript(db, workStagingSchema); // raw_* staging // Two EOP procedures: T1 (contract 90029) already has an EOP annex; T2 (contract 55500) has NONE. diff --git a/packages/db/src/amendments-slice-resolve.test.ts b/packages/db/src/amendments-slice-resolve.test.ts new file mode 100644 index 00000000..931dc449 --- /dev/null +++ b/packages/db/src/amendments-slice-resolve.test.ts @@ -0,0 +1,303 @@ +// Issue #306 (PR #308 review todorkolev "дневните обновявания") — the value-anchor resolver must also run on +// the daily/slice + Worker path, not only the full derive. `scripts/refresh-slice.sql` carries a corpus-safe +// resolver whose candidate contracts come from the served `contracts` table (the whole corpus) UNIONed with +// the current window's raw_contracts, so "unique on the procedure" is asked corpus-wide — closing the gap that +// kept the full-path resolver off the slice (windowed raw_contracts could only answer "unique in the window"). +// +// These tests run the REAL refresh-slice.sql against SQLite via the sqlite3 CLI, exactly as the CLI slice path +// and the Worker compose it: a first window promotes a served corpus, then a later window brings a +// namespace-mismatched annex (its annex-side number is in a different namespace than the contract's filing +// number). The annex links by the exact value_before → signing_value anchor, keeps its provenance, and rolls +// onto the prior-window target — while a corpus-ambiguous annex that would look unique in the window stays +// honestly unlinked. +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); +const migrations = [ + 'packages/db/migrations/0000_init.sql', + 'packages/db/migrations/0001_flow_pairs_bidder_index.sql', + 'packages/db/migrations/0002_current_value_currency.sql', + 'packages/db/migrations/0003_related_persons_foundation.sql', + // #305: refresh-slice.sql writes value_restated/value_treatment/value_suspect into served amendments. + 'packages/db/migrations/0006_amendment_restated.sql', + 'packages/db/migrations/0007_amendment_value_suspect.sql', + 'packages/db/migrations/0008_amendment_provenance.sql', +].map((p) => resolve(root, p)); +const workStagingSchema = resolve(root, 'scripts/work-staging-schema.sql'); +const refreshSlice = resolve(root, 'scripts/refresh-slice.sql'); + +function readScript(dbPath: string, path: string): void { + execFileSync('sqlite3', [dbPath], { + input: `PRAGMA foreign_keys=ON;\n.read ${path}\n`, + stdio: 'pipe', + }); +} + +function sqlite(dbPath: string, sql: string): void { + execFileSync('sqlite3', [dbPath], { input: sql, encoding: 'utf8', stdio: 'pipe' }); +} + +function sqliteJson(dbPath: string, sql: string): T[] { + const out = execFileSync('sqlite3', ['-json', dbPath, sql], { encoding: 'utf8' }).trim(); + return out ? (JSON.parse(out) as T[]) : []; +} + +// Fresh transient staging for a new window, exactly as import.mjs / the Worker do between refreshes. +function resetStaging(dbPath: string): void { + const rows = sqliteJson<{ name: string }>( + dbPath, + "SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'raw_%' ORDER BY name DESC", + ); + for (const row of rows) sqlite(dbPath, `DROP TABLE IF EXISTS "${row.name}";`); + readScript(dbPath, workStagingSchema); +} + +// One EOP procedure header (a tender) so contract promotion has an authority + tender to attach to. +function seedTender(dbPath: string, unp: string, authorityEik: string): void { + sqlite( + dbPath, + `INSERT INTO raw_tenders + (source, dataset_year, fetched_at, unp, tender_id, procedure_type, procurement_subject, + cpv_code, cpv_description, contract_kind, estimated_value, currency, authority_name, + authority_eik, authority_type, published_at) + VALUES + ('eop:tenders:2026-06-01', 2026, '2026-06-07T00:00:00Z', '${unp}', 'T-${unp}', 'open', + 'Subject ${unp}', '45000000', 'Construction', 'works', 5000, 'BGN', 'Authority ${unp}', + '${authorityEik}', 'public', '2026-06-01');`, + ); +} + +// An EOP base contract. `cnum` is the contract's filing number; `signing` its signing value. +function seedContract( + dbPath: string, + opts: { unp: string; cnum: string; signing: number; authorityEik: string; contractorEik: string }, +): void { + const { unp, cnum, signing, authorityEik, contractorEik } = opts; + sqlite( + dbPath, + `INSERT INTO raw_contracts + (source, dataset_year, dataset_variant, fetched_at, needs_enrichment, document_number, + published_at, unp, tender_ext_id, procedure_type, procurement_subject, cpv_code, + cpv_description, contract_kind, estimated_value, procurement_currency, authority_name, + authority_eik, authority_type, contract_number, contract_date, signing_value, currency, + contract_subject, awarded_to_group, contractor_eik, contractor_name) + VALUES + ('eop:contracts:2026-06-01', 2026, 'eop', '2026-06-07T00:00:00Z', 0, 'DOC-${cnum}', + '2026-06-01', '${unp}', 'T-${unp}', 'open', 'Subject ${unp}', '45000000', 'Construction', + 'works', 5000, 'BGN', 'Authority ${unp}', '${authorityEik}', 'public', '${cnum}', + '2026-06-02', ${signing}, 'BGN', 'Contract ${cnum}', 0, '${contractorEik}', 'Bidder ${unp}');`, + ); +} + +// A namespace-mismatched EOP annex: `annexCnum` is the annex-side internal number (matches no contract by +// number); it must link to a contract by the value_before anchor instead. +function seedAnnex( + dbPath: string, + opts: { + unp: string; + annexCnum: string; + valueBefore: number; + valueAfter: number; + authorityEik: string; + contractorEik: string; + }, +): void { + const { unp, annexCnum, valueBefore, valueAfter, authorityEik, contractorEik } = opts; + sqlite( + dbPath, + `INSERT INTO raw_amendments + (source, dataset_year, dataset_variant, fetched_at, seq_no, document_number, contract_number, + contract_date, published_at, unp, authority_eik, authority_name, procurement_subject, + contract_kind, value_before, value_after, value_delta, currency, contractor_eik, description) + VALUES + ('eop:annexes:2026-06-08', 2026, 'eop', '2026-06-08T00:00:00Z', '1', 'AMD-${annexCnum}', + '${annexCnum}', '2026-06-02', '2026-06-09', '${unp}', '${authorityEik}', 'Authority ${unp}', + 'Subject ${unp}', 'works', ${valueBefore}, ${valueAfter}, ${valueAfter - valueBefore}, 'BGN', + '${contractorEik}', 'Namespace-mismatched annex');`, + ); +} + +let dir: string; +let db: string; + +beforeEach(() => { + dir = mkdtempSync(resolve(tmpdir(), 'amendments-slice-resolve-')); + db = resolve(dir, 'work.sqlite'); + for (const m of migrations) readScript(db, m); + readScript(db, workStagingSchema); +}); + +afterEach(() => { + rmSync(dir, { recursive: true, force: true }); +}); + +interface AmendmentRow { + contract_number: string; + contract_number_raw: string | null; + link_method: string | null; +} +interface ContractRow { + contract_number: string; + annex_count: number; + current_value: number | null; +} + +describe('refresh-slice #306 value-anchor resolver', () => { + it('links a window annex to a prior-window served contract by the corpus value anchor', () => { + // Window 1: the base contract, filing number Д-226, is served on its own. + seedTender(db, 'UNP-NS', '123456786'); + seedContract(db, { + unp: 'UNP-NS', + cnum: 'Д-226', + signing: 1000, + authorityEik: '123456786', + contractorEik: '987654308', + }); + readScript(db, refreshSlice); + + // Window 2: only the annex arrives, carrying the internal number 148846 — matches no contract by number, + // but its value_before (1000) is the served contract's signing_value. + resetStaging(db); + seedAnnex(db, { + unp: 'UNP-NS', + annexCnum: '148846', + valueBefore: 1000, + valueAfter: 1200, + authorityEik: '123456786', + contractorEik: '987654308', + }); + readScript(db, refreshSlice); + + const amendments = sqliteJson( + db, + "SELECT contract_number, contract_number_raw, link_method FROM amendments WHERE unp='UNP-NS'", + ); + // Rewritten onto the target contract's filing number, with the annex-side number kept as provenance. + expect(amendments).toEqual([ + { contract_number: 'Д-226', contract_number_raw: '148846', link_method: 'value_anchor' }, + ]); + + // The prior-window target was touched and re-rolled: the annex now counts and drives current_value. + const contract = sqliteJson( + db, + "SELECT contract_number, annex_count, current_value FROM contracts WHERE tender_id='t:UNP-NS'", + ); + expect(contract).toEqual([{ contract_number: 'Д-226', annex_count: 1, current_value: 1200 }]); + }); + + it('leaves a corpus-ambiguous annex unlinked even when it is unique within the window', () => { + // Window 1: two served contracts on the SAME procedure, both signing_value 1000 — the annex value cannot + // pick between them across the corpus. + seedTender(db, 'UNP-AMB', '123456786'); + seedContract(db, { + unp: 'UNP-AMB', + cnum: 'Д-1', + signing: 1000, + authorityEik: '123456786', + contractorEik: '987654308', + }); + seedContract(db, { + unp: 'UNP-AMB', + cnum: 'Д-2', + signing: 1000, + authorityEik: '123456786', + contractorEik: '987654308', + }); + readScript(db, refreshSlice); + + // Window 2: a THIRD matching contract Д-3 (1000) plus the mismatched annex. A windowed-only resolver would + // see just Д-3 and mislink (n_match = 1); the corpus-aware resolver sees Д-1/Д-2/Д-3 and refuses. + resetStaging(db); + seedContract(db, { + unp: 'UNP-AMB', + cnum: 'Д-3', + signing: 1000, + authorityEik: '123456786', + contractorEik: '987654308', + }); + seedAnnex(db, { + unp: 'UNP-AMB', + annexCnum: '148846', + valueBefore: 1000, + valueAfter: 1200, + authorityEik: '123456786', + contractorEik: '987654308', + }); + readScript(db, refreshSlice); + + // Unlinked: the annex keeps its raw annex-side number and no link_method. + const amendments = sqliteJson( + db, + "SELECT contract_number, contract_number_raw, link_method FROM amendments WHERE unp='UNP-AMB'", + ); + expect(amendments).toEqual([ + { contract_number: '148846', contract_number_raw: null, link_method: null }, + ]); + + // None of the three candidate contracts absorbed the annex. + const amended = sqliteJson<{ n: number }>( + db, + "SELECT COUNT(*) AS n FROM contracts WHERE tender_id='t:UNP-AMB' AND annex_count > 0", + ); + expect(amended).toEqual([{ n: 0 }]); + }); + + it('keeps an annex on its zero-value contract it matches BY NUMBER (never value-links to a neighbour)', () => { + // The full path excludes an annex whose (unp, contract_number) is a real contract, regardless of that + // contract's value. The slice path must agree: Д-1 has signing_value 0, Д-2 has 5000; the annex is numbered + // Д-1 with value_before 5000. It matches Д-1 by number, so it must stay on Д-1 — NOT get value-linked to Д-2 + // just because Д-1 fails the signing_value > 0 candidate filter (review todorkolev: the paths must agree). + seedTender(db, 'UNP-ZERO', '123456786'); + seedContract(db, { + unp: 'UNP-ZERO', + cnum: 'Д-2', + signing: 5000, + authorityEik: '123456786', + contractorEik: '987654308', + }); + readScript(db, refreshSlice); + + // Window 2: the zero-value contract Д-1 arrives with the annex that carries its number. + resetStaging(db); + seedContract(db, { + unp: 'UNP-ZERO', + cnum: 'Д-1', + signing: 0, + authorityEik: '123456786', + contractorEik: '987654308', + }); + seedAnnex(db, { + unp: 'UNP-ZERO', + annexCnum: 'Д-1', + valueBefore: 5000, + valueAfter: 9000, + authorityEik: '123456786', + contractorEik: '987654308', + }); + readScript(db, refreshSlice); + + // The annex stays on Д-1 by number — not rewritten to the value-neighbour Д-2. + const amendments = sqliteJson( + db, + "SELECT contract_number, contract_number_raw, link_method FROM amendments WHERE unp='UNP-ZERO'", + ); + expect(amendments).toEqual([ + { contract_number: 'Д-1', contract_number_raw: null, link_method: null }, + ]); + + // Д-1 absorbs the annex; the value-neighbour Д-2 is untouched. + const perContract = sqliteJson<{ contract_number: string; annex_count: number }>( + db, + "SELECT contract_number, annex_count FROM contracts WHERE tender_id='t:UNP-ZERO' ORDER BY contract_number", + ); + expect(perContract).toEqual([ + { contract_number: 'Д-1', annex_count: 1 }, + { contract_number: 'Д-2', annex_count: 0 }, + ]); + }); +}); diff --git a/packages/db/src/amendments-sql.test.ts b/packages/db/src/amendments-sql.test.ts index 75ceea4b..e4f56dfa 100644 --- a/packages/db/src/amendments-sql.test.ts +++ b/packages/db/src/amendments-sql.test.ts @@ -20,6 +20,8 @@ const migration0 = resolve(root, 'packages/db/migrations/0000_init.sql'); const migration6 = resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'); // #305 residual: AMENDMENTS_SQL also selects am.value_suspect, added by this migration. const migration7 = resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'); +// #306 provenance columns on served `amendments` — promote/refresh-slice write contract_number_raw + link_method. +const migration8 = resolve(root, 'packages/db/migrations/0008_amendment_provenance.sql'); function sqlite(dbPath: string, sql: string): string { return execFileSync('sqlite3', [dbPath], { input: sql, encoding: 'utf8' }); @@ -71,6 +73,7 @@ function withDb(fn: (dbPath: string) => T): T { readScript(dbPath, migration0); readScript(dbPath, migration6); readScript(dbPath, migration7); + readScript(dbPath, migration8); return fn(dbPath); } finally { rmSync(dir, { recursive: true, force: true }); diff --git a/packages/db/src/amendments-total-restated.test.ts b/packages/db/src/amendments-total-restated.test.ts index 5909f710..2eee4fad 100644 --- a/packages/db/src/amendments-total-restated.test.ts +++ b/packages/db/src/amendments-total-restated.test.ts @@ -25,6 +25,8 @@ const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_ const migration3Path = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); const migration6Path = resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'); const migration7Path = resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'); +// #306 provenance columns on served `amendments` — promote/refresh-slice write contract_number_raw + link_method. +const migration8Path = resolve(root, 'packages/db/migrations/0008_amendment_provenance.sql'); const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); const derivePath = resolve(root, 'scripts/derive-amendments.sql'); const normalizePath = resolve(root, 'scripts/normalize-raw.sql'); @@ -63,6 +65,7 @@ function withEtlDb(label: string, run: (dbPath: string) => void): void { readScript(dbPath, migration3Path); readScript(dbPath, migration6Path); readScript(dbPath, migration7Path); + readScript(dbPath, migration8Path); readScript(dbPath, stagingPath); run(dbPath); } finally { diff --git a/packages/db/src/amendments-total-suspect.test.ts b/packages/db/src/amendments-total-suspect.test.ts index 2fa3c913..e34fdbab 100644 --- a/packages/db/src/amendments-total-suspect.test.ts +++ b/packages/db/src/amendments-total-suspect.test.ts @@ -26,6 +26,8 @@ const migration3Path = resolve(root, 'packages/db/migrations/0003_related_person const migration6Path = resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'); // #305 residual: served amendments gained value_suspect (promote + refresh-slice write it). const migration7Path = resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'); +// #306 provenance columns on served `amendments` — promote/refresh-slice write contract_number_raw + link_method. +const migration8Path = resolve(root, 'packages/db/migrations/0008_amendment_provenance.sql'); const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); const derivePath = resolve(root, 'scripts/derive-amendments.sql'); const normalizePath = resolve(root, 'scripts/normalize-raw.sql'); @@ -68,6 +70,7 @@ function withEtlDb(label: string, run: (dbPath: string) => void): void { readScript(dbPath, migration3Path); readScript(dbPath, migration6Path); readScript(dbPath, migration7Path); + readScript(dbPath, migration8Path); readScript(dbPath, stagingPath); run(dbPath); } finally { diff --git a/packages/db/src/contractor-identity-sql.test.ts b/packages/db/src/contractor-identity-sql.test.ts index e47c775a..b46d01bb 100644 --- a/packages/db/src/contractor-identity-sql.test.ts +++ b/packages/db/src/contractor-identity-sql.test.ts @@ -26,6 +26,11 @@ const migration7 = readFileSync( resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'), 'utf8', ); +// #306 provenance columns on served `amendments` — promote/refresh-slice write contract_number_raw + link_method. +const migration8 = readFileSync( + resolve(root, 'packages/db/migrations/0008_amendment_provenance.sql'), + 'utf8', +); const staging = readFileSync(resolve(root, 'scripts/work-staging-schema.sql'), 'utf8'); const normalize = readFileSync(resolve(root, 'scripts/normalize-raw.sql'), 'utf8'); const precompute = readFileSync(resolve(root, 'scripts/precompute.sql'), 'utf8'); @@ -79,6 +84,7 @@ function build(path: 'normalize' | 'refresh'): DatabaseSync { db.exec(migration3); db.exec(migration6); db.exec(migration7); + db.exec(migration8); db.exec(staging); db.exec(seed); if (path === 'normalize') { diff --git a/packages/db/src/etl-entity-canonicalization-sql.test.ts b/packages/db/src/etl-entity-canonicalization-sql.test.ts index 7a682ffa..53012248 100644 --- a/packages/db/src/etl-entity-canonicalization-sql.test.ts +++ b/packages/db/src/etl-entity-canonicalization-sql.test.ts @@ -14,6 +14,8 @@ const migration3Path = resolve(root, 'packages/db/migrations/0003_related_person // #305 Tier-2: served amendments gained value_restated/value_treatment (promote + refresh-slice write them). const migration6Path = resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'); const migration7Path = resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'); +// #306 provenance columns on served `amendments` — promote/refresh-slice write contract_number_raw + link_method. +const migration8Path = resolve(root, 'packages/db/migrations/0008_amendment_provenance.sql'); const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); const etlPaths = [ ['normalize-raw', resolve(root, 'scripts/normalize-raw.sql')], @@ -45,6 +47,7 @@ function withEtlDb(label: string, run: (dbPath: string) => void): void { readScript(dbPath, migration3Path); readScript(dbPath, migration6Path); readScript(dbPath, migration7Path); + readScript(dbPath, migration8Path); readScript(dbPath, stagingPath); run(dbPath); } finally { diff --git a/packages/db/src/refresh-slice.test.ts b/packages/db/src/refresh-slice.test.ts index 3685e1e4..32ecd676 100644 --- a/packages/db/src/refresh-slice.test.ts +++ b/packages/db/src/refresh-slice.test.ts @@ -17,6 +17,8 @@ const migration3Path = resolve(root, 'packages/db/migrations/0003_related_person const migration6Path = resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'); // #305 residual: served amendments gained value_suspect (refresh-slice promotes it). const migration7Path = resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'); +// #306 provenance columns on served `amendments` — promote/refresh-slice write contract_number_raw + link_method. +const migration8Path = resolve(root, 'packages/db/migrations/0008_amendment_provenance.sql'); const refreshSlicePath = resolve(root, 'scripts/refresh-slice.sql'); const normalizePath = resolve(root, 'scripts/normalize-raw.sql'); const deriveAmendmentsPath = resolve(root, 'scripts/derive-amendments.sql'); @@ -191,6 +193,7 @@ function initWorkDb(dbPath: string): void { readScript(dbPath, migration3Path); readScript(dbPath, migration6Path); readScript(dbPath, migration7Path); + readScript(dbPath, migration8Path); readScript(dbPath, workStagingSchemaPath); } @@ -584,6 +587,7 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration3Path); readScript(dbPath, migration6Path); readScript(dbPath, migration7Path); + readScript(dbPath, migration8Path); readScript(dbPath, workStagingSchemaPath); seedEopBaseDay(dbPath); @@ -668,6 +672,7 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration3Path); readScript(dbPath, migration6Path); readScript(dbPath, migration7Path); + readScript(dbPath, migration8Path); readScript(dbPath, workStagingSchemaPath); // An EOP procedure (tender + base contract) with УНП UNP-SLICE / tender.id TENDER-SLICE. An @@ -853,6 +858,7 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration3Path); readScript(dbPath, migration6Path); readScript(dbPath, migration7Path); + readScript(dbPath, migration8Path); readScript(dbPath, workStagingSchemaPath); seedEopOnlySharedNumber(dbPath); readScript(dbPath, refreshSlicePath); @@ -906,6 +912,7 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration3Path); readScript(dbPath, migration6Path); readScript(dbPath, migration7Path); + readScript(dbPath, migration8Path); readScript(dbPath, workStagingSchemaPath); sqlite( dbPath, @@ -959,6 +966,7 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration3Path); readScript(dbPath, migration6Path); readScript(dbPath, migration7Path); + readScript(dbPath, migration8Path); readScript(dbPath, workStagingSchemaPath); sqlite( dbPath, @@ -1098,6 +1106,7 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration3Path); readScript(dbPath, migration6Path); readScript(dbPath, migration7Path); + readScript(dbPath, migration8Path); readScript(dbPath, workStagingSchemaPath); sqlite( dbPath, diff --git a/packages/db/src/value-flag-annex-step-sql.test.ts b/packages/db/src/value-flag-annex-step-sql.test.ts index bfa8af32..c4872177 100644 --- a/packages/db/src/value-flag-annex-step-sql.test.ts +++ b/packages/db/src/value-flag-annex-step-sql.test.ts @@ -29,6 +29,8 @@ const migration3Path = resolve(root, 'packages/db/migrations/0003_related_person // #305 Tier-2: served amendments gained value_restated/value_treatment (promote + refresh-slice write them). const migration6Path = resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'); const migration7Path = resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'); +// #306 provenance columns on served `amendments` — promote/refresh-slice write contract_number_raw + link_method. +const migration8Path = resolve(root, 'packages/db/migrations/0008_amendment_provenance.sql'); const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); const derivePath = resolve(root, 'scripts/derive-amendments.sql'); const promotePath = resolve(root, 'scripts/promote-amendments.sql'); @@ -65,6 +67,7 @@ function withEtlDb(label: string, run: (dbPath: string) => void): void { readScript(dbPath, migration3Path); readScript(dbPath, migration6Path); readScript(dbPath, migration7Path); + readScript(dbPath, migration8Path); readScript(dbPath, stagingPath); run(dbPath); } finally { diff --git a/packages/db/src/value-flag-stotinki-sql.test.ts b/packages/db/src/value-flag-stotinki-sql.test.ts index d2fe07e5..a13808aa 100644 --- a/packages/db/src/value-flag-stotinki-sql.test.ts +++ b/packages/db/src/value-flag-stotinki-sql.test.ts @@ -23,6 +23,8 @@ const migration3Path = resolve(root, 'packages/db/migrations/0003_related_person // #305 Tier-2: served amendments gained value_restated/value_treatment (promote + refresh-slice write them). const migration6Path = resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'); const migration7Path = resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'); +// #306 provenance columns on served `amendments` — promote/refresh-slice write contract_number_raw + link_method. +const migration8Path = resolve(root, 'packages/db/migrations/0008_amendment_provenance.sql'); const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); const etlPaths = [ ['normalize-raw', resolve(root, 'scripts/normalize-raw.sql')], @@ -54,6 +56,7 @@ function withEtlDb(label: string, run: (dbPath: string) => void): void { readScript(dbPath, migration3Path); readScript(dbPath, migration6Path); readScript(dbPath, migration7Path); + readScript(dbPath, migration8Path); readScript(dbPath, stagingPath); run(dbPath); } finally { diff --git a/packages/ingest/src/refresh.ts b/packages/ingest/src/refresh.ts index 45808bf3..0aed14ad 100644 --- a/packages/ingest/src/refresh.ts +++ b/packages/ingest/src/refresh.ts @@ -98,6 +98,12 @@ const LEGACY_TRANSIENT_STAGING_TABLES = [ 'raw_egov_amendments', ] as const; +// Scratch tables that live only for the span of a single derive step. Each is DROP-guarded at the top of +// its own script, so it self-heals on the next run; listing it here also sweeps it after an aborted run so +// it never lingers in D1 (review nikimilenkov LOW 2 — #306's value-resolver scratch table). Not part of +// work-staging-schema.sql, so it stays out of transientStagingStatements' recreate path. +const SCRATCH_TABLES = ['amendment_contract_resolve'] as const; + function touchesTransientStaging(statement: string): boolean { return TRANSIENT_STAGING_TABLES.some((table) => statement.includes(table)); } @@ -148,7 +154,7 @@ export function fullClearTables(normalizeRawSql: string): string[] { } export function dropTransientStagingStatements(): string[] { - return [...TRANSIENT_STAGING_TABLES, ...LEGACY_TRANSIENT_STAGING_TABLES] + return [...SCRATCH_TABLES, ...TRANSIENT_STAGING_TABLES, ...LEGACY_TRANSIENT_STAGING_TABLES] .reverse() .map((table) => `DROP TABLE IF EXISTS ${table}`); } diff --git a/scripts/derive-amendments.sql b/scripts/derive-amendments.sql index 23dcee7a..217490ea 100644 --- a/scripts/derive-amendments.sql +++ b/scripts/derive-amendments.sql @@ -1,8 +1,9 @@ -- Sigma — roll raw_amendments up onto raw_contracts. -- Run AFTER scripts/load-eop.mjs (which stages the EOP base + in-bucket OCDS amendments). --- Re-runnable. First recovers the УНП for OCDS amendments via the tender.id bridge (#286), then --- prefers the EOP annex over its OCDS twin (dropping the twin), then matches amendments by --- (unp, contract_number). +-- Re-runnable. On the full-derive path scripts/resolve-amendment-contracts.sql (#306) has already rewritten +-- namespace-mismatched EOP annexes onto their target contract_number BEFORE this file runs. This file then +-- recovers the УНП for OCDS amendments via the tender.id bridge (#286), prefers the EOP annex over its OCDS +-- twin (dropping the twin), and matches amendments by (unp, contract_number). -- current_value = the after-value of the LATEST amendment; annex_count = how many. -- Contracts without amendments keep annex_count = 0 and current_value = NULL (the -- convention downstream is COALESCE(current_value, signing_value)). @@ -110,12 +111,24 @@ WHERE source LIKE 'ocds:%' AND e.contract_number = raw_amendments.contract_number ); +-- #306: the value-anchor resolver that links namespace-mismatched EOP annexes (annex-side number ≠ the +-- contract's filing number) by rewriting raw_amendments.contract_number lives in +-- scripts/resolve-amendment-contracts.sql and runs BEFORE this file on the full-derive path only (see that +-- file's header for the ordering-blocker and slice-gating rationale — reviews todorkolev #1/#3, nikimilenkov +-- HIGH 1). By the time this rollup runs, resolved annexes already carry their target contract_number (and +-- keep the original annex number in contract_number_raw for the natural_key below), so the rollup links them +-- with no further change. + UPDATE raw_contracts SET annex_count = 0, current_value = NULL; WITH keyed AS ( SELECT *, - 'am:' || COALESCE(unp, '') || ':' || COALESCE(contract_number, '') || ':' || + -- #306: key on the ORIGINAL annex-side number (contract_number_raw) when the value resolver rewrote a + -- row, so a resolved annex keeps a stable identity and never collides with a native annex that shares + -- document_number on the target contract (review nikimilenkov MEDIUM 3). NULL raw → the plain number, + -- so unresolved rows and the slice path (which never resolves) produce byte-identical keys. + 'am:' || COALESCE(unp, '') || ':' || COALESCE(NULLIF(contract_number_raw, ''), contract_number, '') || ':' || COALESCE( NULLIF(document_number, ''), NULLIF(correction_number, ''), diff --git a/scripts/import.mjs b/scripts/import.mjs index a841759c..fe2aba16 100644 --- a/scripts/import.mjs +++ b/scripts/import.mjs @@ -285,6 +285,12 @@ function assertDeriveWindowSafe(mode, from) { } async function runFullDerive() { + // #306: link namespace-mismatched EOP annexes by value BEFORE derive-amendments.sql (its prefer-EOP + // dedup would otherwise resurrect OCDS twins — review todorkolev #1). This STANDALONE script is the + // full-derive form (candidates from the whole re-staged raw_contracts). The daily/slice + Worker path runs + // an equivalent value anchor inside refresh-slice.sql that draws candidates from the served `contracts` + // corpus instead of the window (so "unique on the procedure" is corpus-wide — review nikimilenkov HIGH 1). + execSql(resolve(root, 'scripts/resolve-amendment-contracts.sql')); execSql(resolve(root, 'scripts/derive-amendments.sql')); run('node', ['scripts/load-fx.mjs', '--apply', ...passthru]); execSql(resolve(root, 'scripts/load-nuts.sql')); @@ -362,6 +368,8 @@ async function runWorkBackfill() { `--out=${resolve(workDir, `${stem}.eop-load.sql`)}`, ...loadFlags, ]); + // #306: value-anchor resolver runs first on the full-derive path — see runFullDerive. + sqliteFile(workDb, resolve(root, 'scripts/resolve-amendment-contracts.sql')); sqliteFile(workDb, resolve(root, 'scripts/derive-amendments.sql')); run('node', [ 'scripts/load-fx.mjs', diff --git a/scripts/normalize-raw.sql b/scripts/normalize-raw.sql index c5ba2c72..1c1f7a11 100644 --- a/scripts/normalize-raw.sql +++ b/scripts/normalize-raw.sql @@ -732,7 +732,10 @@ INSERT OR IGNORE INTO contracts -- travels alongside the value it minted — computed ONCE over raw_amendments, not per-row). WITH amendment_dedup AS ( SELECT *, - 'am:' || COALESCE(unp, '') || ':' || COALESCE(contract_number, '') || ':' || + -- #306: key on the original annex-side number (contract_number_raw) when the value resolver rewrote a + -- row — must match derive-amendments.sql / promote-amendments.sql byte-for-byte (review nikimilenkov + -- MEDIUM 3). NULL raw → the plain number, so unresolved rows are unaffected. + 'am:' || COALESCE(unp, '') || ':' || COALESCE(NULLIF(contract_number_raw, ''), contract_number, '') || ':' || COALESCE( NULLIF(document_number, ''), NULLIF(correction_number, ''), @@ -1278,7 +1281,10 @@ CREATE TABLE IF NOT EXISTS pipeline_stats ( DELETE FROM pipeline_stats; WITH amendment_dedup AS ( SELECT *, - 'am:' || COALESCE(unp, '') || ':' || COALESCE(contract_number, '') || ':' || + -- #306: key on the original annex-side number (contract_number_raw) when the value resolver rewrote a + -- row — must match derive-amendments.sql / promote-amendments.sql byte-for-byte (review nikimilenkov + -- MEDIUM 3). NULL raw → the plain number, so unresolved rows are unaffected. + 'am:' || COALESCE(unp, '') || ':' || COALESCE(NULLIF(contract_number_raw, ''), contract_number, '') || ':' || COALESCE( NULLIF(document_number, ''), NULLIF(correction_number, ''), diff --git a/scripts/promote-amendments.sql b/scripts/promote-amendments.sql index b7822315..c5e70557 100644 --- a/scripts/promote-amendments.sql +++ b/scripts/promote-amendments.sql @@ -7,13 +7,18 @@ DELETE FROM amendments; INSERT OR REPLACE INTO amendments ( - id, natural_key, contract_number, unp, value_before, value_after, value_delta, currency, - published_at, document_number, description, source, value_restated, value_treatment, value_suspect + id, natural_key, contract_number, contract_number_raw, link_method, unp, + value_before, value_after, value_delta, currency, + published_at, document_number, description, source, + value_restated, value_treatment, value_suspect ) WITH keyed AS ( SELECT *, - 'am:' || COALESCE(unp, '') || ':' || COALESCE(contract_number, '') || ':' || + -- #306: key on the original annex-side number (contract_number_raw) when the value resolver rewrote a + -- row — MUST match derive-amendments.sql's rollup key byte-for-byte, else annex_count and the served + -- timeline disagree (review nikimilenkov MEDIUM 3). NULL raw → the plain number. + 'am:' || COALESCE(unp, '') || ':' || COALESCE(NULLIF(contract_number_raw, ''), contract_number, '') || ':' || COALESCE( NULLIF(document_number, ''), NULLIF(correction_number, ''), @@ -38,6 +43,8 @@ SELECT natural_key, natural_key, contract_number, + contract_number_raw, -- #306 provenance: the annex-side number before the value resolver rewrote it + link_method, -- #306 provenance: 'value_anchor' for value-linked rows, else NULL unp, value_before, -- #305 Tier-2: serve the effective (text-corrected) after and a self-consistent delta; a restated diff --git a/scripts/refresh-slice.sql b/scripts/refresh-slice.sql index cdaf8527..5bd90796 100644 --- a/scripts/refresh-slice.sql +++ b/scripts/refresh-slice.sql @@ -21,6 +21,7 @@ DROP TABLE IF EXISTS refresh_unp_prefix_authorities; DROP TABLE IF EXISTS refresh_joint_authority_members; DROP TABLE IF EXISTS refresh_joint_tender_sources; DROP TABLE IF EXISTS refresh_amendment_winners; +DROP TABLE IF EXISTS refresh_amendment_contract_resolve; CREATE TABLE refresh_touched_contracts (id TEXT PRIMARY KEY); CREATE TABLE refresh_touched_bidders (bidder_id TEXT PRIMARY KEY); CREATE TABLE refresh_touched_authorities (authority_id TEXT PRIMARY KEY); @@ -64,6 +65,155 @@ WHERE source LIKE 'ocds:%' ); -- @bridge-lockstep end +-- #306: slice-safe value-anchor resolver. Links EOP annexes whose annex-side number is in a different +-- namespace than the contract's filing number (annex carries an internal number like 148846; the contract +-- carries Д-226), by the exact value_before → signing_value anchor — the same 99.99%-precision link the +-- full path applies in scripts/resolve-amendment-contracts.sql. It runs HERE, before the prefer-EOP dedup +-- DELETE below and the amendment promotion further down, for the SAME reason the full path runs it before +-- derive-amendments.sql's dedup: rewriting an EOP annex onto a contract that already kept an OCDS twin would +-- resurrect the twin and trip amendment-twin-dedup (#303, review todorkolev #1). +-- +-- The full path is gated OFF the slice because its candidates came from the WINDOWED raw_contracts, where +-- "unique on the procedure" means "unique in the window" and a corpus-ambiguous annex would mislink (review +-- nikimilenkov HIGH 1). This slice version closes that gap the way the deferred plan §4 prescribes: candidate +-- contracts are drawn from the CUMULATIVE served `contracts` (the whole corpus) UNIONed with this window's +-- raw_contracts, so uniqueness is asked over the corpus, not the window — the measured precision carries. It +-- is intentionally NOT under a byte-identical lockstep marker with the full-path script: the candidate source +-- differs by construction. Scans are bounded to the procedures that actually have an EOP annex in this window +-- (window_unps), so the served-corpus read stays a keyed lookup, not a full scan (idx_contracts_tender_id). +-- Resolved prior-window targets need no extra touch-wiring: the rewrite below lands the target contract_number +-- on raw_amendments, and the existing `@refresh-batch amendments` touch join (raw_amendments → contracts on +-- contract_number) then scopes those targets into refresh_touched_contracts for free (plan §4). +DROP TABLE IF EXISTS refresh_amendment_contract_resolve; +CREATE TABLE refresh_amendment_contract_resolve AS +WITH window_unps AS ( + SELECT DISTINCT unp FROM raw_amendments + WHERE source LIKE 'eop:%' AND unp IS NOT NULL AND contract_number IS NOT NULL +), +-- Every contract NUMBER that exists on the affected procedures — from BOTH this window's raw_contracts and the +-- served corpus, REGARDLESS of signing_value. `grp` below asks this to decide "is the annex's number already a +-- real contract on the procedure". It MUST be value-agnostic: the full path asks only "does such a contract +-- exist" (NOT EXISTS over raw_contracts), so keying the slice question off contract_candidates (which requires +-- signing_value > 0) would diverge — an annex that points to a ZERO-value contract by number would look +-- namespace-mismatched and get value-linked to a neighbour, contradicting "links by number, not value" and +-- making the two paths disagree (review todorkolev). +all_contract_numbers AS ( + SELECT unp, contract_number FROM raw_contracts + WHERE contract_number IS NOT NULL AND unp IN (SELECT unp FROM window_unps) + UNION + SELECT substr(tender_id, 3) AS unp, contract_number FROM contracts + WHERE contract_number IS NOT NULL AND tender_id IN (SELECT 't:' || unp FROM window_unps) +), +-- One row per LOGICAL contract on the affected procedures, for VALUE matching (signing_value > 0). Two sources, +-- deduped: (0) this window's raw_contracts — cumulative EOP buckets repeat a contract across days, so collapse +-- to the latest source-day then highest id, mirroring normalize-raw; (1) the served `contracts` corpus (unp = +-- the tender_id suffix, contractor ЕИК via the winning bidder). A contract present in BOTH is one logical +-- contract — src_rank keeps the window row (freshest signing_value) and the COUNT below never double-counts it +-- into a false ambiguity. +contract_candidates AS ( + SELECT unp, contract_number, signing_value, currency, contractor_eik FROM ( + SELECT unp, contract_number, signing_value, currency, contractor_eik, + ROW_NUMBER() OVER ( + PARTITION BY unp, contract_number ORDER BY src_rank, cand_source DESC, ord DESC + ) AS rn + FROM ( + SELECT c.unp, c.contract_number, c.signing_value, c.currency, + TRIM(CASE WHEN c.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(c.contractor_eik, 5) ELSE c.contractor_eik END) + AS contractor_eik, + 0 AS src_rank, c.source AS cand_source, c.id AS ord + FROM raw_contracts c + WHERE c.contract_number IS NOT NULL + AND c.signing_value IS NOT NULL AND c.signing_value > 0 + AND c.unp IN (SELECT unp FROM window_unps) + UNION ALL + SELECT substr(c.tender_id, 3) AS unp, c.contract_number, c.signing_value, c.currency, + b.eik_normalized AS contractor_eik, + 1 AS src_rank, '' AS cand_source, '' AS ord + FROM contracts c + LEFT JOIN bidders b ON b.id = c.bidder_id + WHERE c.contract_number IS NOT NULL + AND c.signing_value IS NOT NULL AND c.signing_value > 0 + AND c.tender_id IN (SELECT 't:' || unp FROM window_unps) + ) + ) WHERE rn = 1 +), +-- EOP annexes on the affected procedures that match NO REAL contract by (unp, contract_number) — the +-- namespace-mismatch group. Value-less members (admin/term steps mid-chain) are included so they can inherit +-- the chain's target (review nikimilenkov MEDIUM 2). The NOT EXISTS is over all_contract_numbers (every real +-- contract number on the procedure, value-agnostic), so an annex that DOES match a corpus contract directly is +-- correctly excluded — it links by number, not value — even when that target has signing_value <= 0, matching +-- the full path exactly (review todorkolev). +grp AS ( + SELECT a.id AS amendment_id, a.unp, a.contract_number AS annex_cnum, + a.value_before, a.currency, + TRIM(CASE WHEN a.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(a.contractor_eik, 5) ELSE a.contractor_eik END) + AS contractor_eik + FROM raw_amendments a + WHERE a.source LIKE 'eop:%' + AND a.unp IS NOT NULL AND a.contract_number IS NOT NULL + AND NOT EXISTS ( + SELECT 1 FROM all_contract_numbers c + WHERE c.unp = a.unp AND c.contract_number = a.contract_number + ) +), +-- Exact, currency- and contractor-matched value anchor (< 0.5 стотинка). Currency must be EXPLICIT on both +-- sides (no blank-vs-blank agreement via a default). The EIK guard is null-tolerant and refuses a value +-- collision onto a different contractor's contract for free (review nikimilenkov LOW 1 / MEDIUM 5). +vmatch AS ( + SELECT g.amendment_id, g.unp, g.annex_cnum, c.contract_number AS resolved_cnum, + COUNT(*) OVER (PARTITION BY g.amendment_id) AS n_match + FROM grp g + JOIN contract_candidates c + ON c.unp = g.unp + AND ABS(c.signing_value - g.value_before) < 0.005 + AND NULLIF(c.currency, '') IS NOT NULL + AND NULLIF(g.currency, '') IS NOT NULL + AND c.currency = g.currency + AND (g.contractor_eik IS NULL OR c.contractor_eik IS NULL OR g.contractor_eik = c.contractor_eik) + WHERE g.value_before IS NOT NULL AND g.value_before > 0 +), +-- A member's OWN unique (n_match = 1) exact match — trustworthy on its own, so it always applies and is NOT +-- voided when annex-number siblings point elsewhere (a lot-base number shared across contracts; review +-- nikimilenkov MEDIUM 1). Propagation is withheld on disagreement, never the direct hits. +direct AS ( + SELECT amendment_id, unp, annex_cnum, resolved_cnum FROM vmatch WHERE n_match = 1 +), +-- Propagate one AGREED target across a (unp, annex-number) chain to value-less members, only when the direct +-- members agree on a single contract (review MEDIUM 2). +group_target AS ( + SELECT unp, annex_cnum, MIN(resolved_cnum) AS resolved_cnum + FROM direct GROUP BY unp, annex_cnum HAVING COUNT(DISTINCT resolved_cnum) = 1 +) +-- Link to the own unique match, else the agreed group target — UNLESS the member is itself value-ambiguous +-- (n_match >= 2), which never links directly or by inheritance (review todorkolev #2). +SELECT g.amendment_id, + COALESCE( + (SELECT d.resolved_cnum FROM direct d WHERE d.amendment_id = g.amendment_id), + (SELECT gt.resolved_cnum FROM group_target gt WHERE gt.unp = g.unp AND gt.annex_cnum = g.annex_cnum) + ) AS resolved_cnum +FROM grp g +WHERE NOT EXISTS ( + SELECT 1 FROM vmatch v WHERE v.amendment_id = g.amendment_id AND v.n_match >= 2 +); + +CREATE INDEX idx_refresh_amendment_contract_resolve_id + ON refresh_amendment_contract_resolve(amendment_id); + +-- Rewrite in place, preserving provenance: keep the annex-side number in contract_number_raw and stamp +-- link_method so value-linked rows stay enumerable through promotion into served `amendments`. The raw number +-- also keeps the amendment natural_key stable (see the promotion below and derive/promote on the full path), +-- so a resolved row never collides with a native annex sharing document_number on the target (MEDIUM 3). +UPDATE raw_amendments +SET + contract_number_raw = contract_number, + link_method = 'value_anchor', + contract_number = ( + SELECT r.resolved_cnum FROM refresh_amendment_contract_resolve r WHERE r.amendment_id = raw_amendments.id + ) +WHERE id IN (SELECT amendment_id FROM refresh_amendment_contract_resolve WHERE resolved_cnum IS NOT NULL); + +DROP TABLE IF EXISTS refresh_amendment_contract_resolve; + -- #286: prefer the EOP annex — drop OCDS twins so annex_count and the served timeline aren't doubled. -- The full path (derive-amendments.sql) only checks raw_amendments because it re-stages the whole corpus -- every run. The slice path must ALSO consult the cumulative served `amendments`: an EOP annex promoted by @@ -90,7 +240,7 @@ WHERE source LIKE 'ocds:%' CREATE TABLE refresh_amendment_winners AS WITH keyed AS ( SELECT *, - 'am:' || COALESCE(unp, '') || ':' || COALESCE(contract_number, '') || ':' || + 'am:' || COALESCE(unp, '') || ':' || COALESCE(NULLIF(contract_number_raw, ''), contract_number, '') || ':' || COALESCE( NULLIF(document_number, ''), NULLIF(correction_number, ''), NULLIF(seq_no, ''), 'content:' || COALESCE(published_at, '') || ':' || @@ -1716,13 +1866,15 @@ WHERE source LIKE 'ocds:%' ); INSERT OR REPLACE INTO amendments ( - id, natural_key, contract_number, unp, value_before, value_after, value_delta, currency, - published_at, document_number, description, source, value_restated, value_treatment, value_suspect + id, natural_key, contract_number, contract_number_raw, link_method, unp, + value_before, value_after, value_delta, currency, + published_at, document_number, description, source, + value_restated, value_treatment, value_suspect ) WITH keyed AS ( SELECT *, - 'am:' || COALESCE(unp, '') || ':' || COALESCE(contract_number, '') || ':' || + 'am:' || COALESCE(unp, '') || ':' || COALESCE(NULLIF(contract_number_raw, ''), contract_number, '') || ':' || COALESCE( NULLIF(document_number, ''), NULLIF(correction_number, ''), @@ -1747,6 +1899,8 @@ SELECT natural_key, natural_key, contract_number, + contract_number_raw, -- #306 provenance: the annex-side number before the value resolver rewrote it + link_method, -- #306 provenance: 'value_anchor' for value-linked rows, else NULL unp, value_before, -- #305 Tier-2: serve the effective (text-corrected) after and a self-consistent delta; the current_value diff --git a/scripts/resolve-amendment-contracts.sql b/scripts/resolve-amendment-contracts.sql new file mode 100644 index 00000000..3498505d --- /dev/null +++ b/scripts/resolve-amendment-contracts.sql @@ -0,0 +1,150 @@ +-- Sigma — #306: link EOP annexes whose annex-side number is in a different namespace than the contract +-- number. The annex carries an internal number (e.g. 148846) while the contract on the same procedure +-- carries the buyer's filing number (e.g. Д-226), so the (unp, contract_number) join drops ~7% of EOP +-- annexes out of every annex→contract→company/authority rollup. String normalisation recovers almost none +-- of them (measured), because the two numbers are genuinely unrelated identifiers. Resolve by VALUE +-- instead: an annex's value_before is the contract's value at amendment time, so it equals the target +-- contract's signing_value. Link only when value_before matches EXACTLY (< 0.5 стотинка), currency- and +-- contractor-matched, uniquely one contract on the procedure — measured 99.99% precision on the already- +-- linked corpus (9348/9349). Rows that match 2+ contracts (value-ambiguous) or none (target not yet +-- ingested — the #249 class) are LEFT unlinked: an honest gap beats a wrong contract on a transparency site. +-- +-- FULL-PATH FORM (review nikimilenkov HIGH 1 + todorkolev #3). This file runs from runFullDerive / +-- runWorkBackfill in scripts/import.mjs, BEFORE derive-amendments.sql, with candidates drawn from the whole +-- re-staged raw_contracts. It is deliberately NOT run on the slice path (runSliceDerive): the slice's +-- raw_contracts holds only the current window, so "unique on the procedure" would mean "unique in the window", +-- not in the corpus — a corpus-ambiguous annex would look unique in a narrow window and mislink. The daily/ +-- slice + Worker path instead runs a corpus-safe equivalent INSIDE scripts/refresh-slice.sql (search "#306: +-- slice-safe value-anchor resolver"), whose candidates come from the served `contracts` table plus this +-- window's raw_contracts, so uniqueness is asked corpus-wide and the measured precision carries. Both forms +-- share the same value/currency/EIK anchor, chain rules, and provenance stamping; see +-- docs/implementation-plans/306-amendment-contract-namespace-link.md §4. +-- +-- ORDER (review todorkolev #1, the blocker): this runs BEFORE the #286 prefer-EOP dedup DELETE in +-- derive-amendments.sql. Rewriting an EOP annex onto a contract that already kept an OCDS twin would +-- resurrect that twin (annex_count = 2 on a one-annex contract) and trip the amendment-twin-dedup integrity +-- gate (#303), failing the whole derive. Running first — and above the #286 diagnostics, so their dropped/ +-- excess counts stay honest bounds on what the dedup actually removes — keeps the dedup the sole twin guard. +-- The resolver reads only source LIKE 'eop:%' rows + raw_contracts, so it has no dependency on the OCDS +-- bridge and is safe to run first. + +-- Candidate contracts, deduped to one row per LOGICAL contract. raw_contracts is CUMULATIVE — the EOP daily +-- open-data buckets repeat the same contract across consecutive days, and the collapse to one row per +-- (unp, contract_number) happens later in normalize-raw.sql, NOT in staging (review nikimilenkov HIGH 2). +-- Without this dedup, COUNT(*) OVER below would count staging ROWS, not contracts: a contract present in N +-- daily buckets would read as n_match = N and fail-close every real link, and a superseded row could match +-- an annex to a stale value. Mirror normalize-raw's rule (latest source-day, then highest id, wins). +DROP TABLE IF EXISTS amendment_contract_resolve; +CREATE TABLE amendment_contract_resolve AS +WITH contract_candidates AS ( + SELECT unp, contract_number, signing_value, currency, contractor_eik + FROM ( + SELECT c.unp, c.contract_number, c.signing_value, c.currency, c.contractor_eik, + ROW_NUMBER() OVER ( + PARTITION BY c.unp, c.contract_number + ORDER BY c.source DESC, c.id DESC + ) AS rn + FROM raw_contracts c + WHERE c.contract_number IS NOT NULL + AND c.signing_value IS NOT NULL AND c.signing_value > 0 + ) + WHERE rn = 1 +), +-- Every EOP annex whose (unp, contract_number) matches no contract — the namespace-mismatch group. Includes +-- value-less members (value_before NULL/≤0): an admin/term annex mid-chain carries no signing value of its +-- own but must still inherit the chain's target (review nikimilenkov MEDIUM 2), else the chain breaks and +-- current_value stops early. Grouped by the shared annex-side number (unp, annex_cnum) = one contract's chain. +grp AS ( + SELECT a.id AS amendment_id, a.unp, a.contract_number AS annex_cnum, + a.value_before, a.currency, a.contractor_eik + FROM raw_amendments a + WHERE a.source LIKE 'eop:%' + AND a.unp IS NOT NULL AND a.contract_number IS NOT NULL + AND NOT EXISTS ( + SELECT 1 FROM raw_contracts c + WHERE c.unp = a.unp AND c.contract_number = a.contract_number + ) +), +-- Exact, currency- and contractor-matched value anchor. Only members that carry a usable value_before +-- participate in matching. The currency guard requires an EXPLICIT currency on both sides (review +-- nikimilenkov LOW 1): a blank-vs-blank pair must not silently agree via a 'BGN' default under so tight a +-- gate. The EIK guard (review nikimilenkov MEDIUM 5) is null-tolerant — the annex already carries the +-- contractor, so a value collision onto a DIFFERENT contractor's contract is refused for free. +vmatch AS ( + SELECT g.amendment_id, g.unp, g.annex_cnum, c.contract_number AS resolved_cnum, + COUNT(*) OVER (PARTITION BY g.amendment_id) AS n_match + FROM grp g + JOIN contract_candidates c + ON c.unp = g.unp + AND ABS(c.signing_value - g.value_before) < 0.005 + AND NULLIF(c.currency, '') IS NOT NULL + AND NULLIF(g.currency, '') IS NOT NULL + AND c.currency = g.currency + AND (g.contractor_eik IS NULL OR c.contractor_eik IS NULL OR g.contractor_eik = c.contractor_eik) + WHERE g.value_before IS NOT NULL AND g.value_before > 0 +), +-- `direct` — each member's OWN unique (n_match = 1) exact value match. This is trustworthy on its own +-- (exact cent + unique on the whole procedure = the measured 99.99% precision), so a direct hit always +-- applies. Crucially it is NOT voided when its annex-number siblings point elsewhere: an annex-side number +-- can be a LOT-BASE shared across contracts (real corpus: `20РП-У50А015` → …-Л01 @ 22569.98 AND …-Л03 @ +-- 28557.50, each annex exactly-uniquely matching its own lot). The one disagreement group in the whole live +-- corpus is exactly this benign multi-lot case — refusing it (an earlier revision, review nikimilenkov +-- MEDIUM 1) dropped 2 confirmed-correct links and prevented zero wrong ones, so disagreement withholds +-- PROPAGATION only, never the direct hits themselves. +direct AS ( + SELECT amendment_id, unp, annex_cnum, resolved_cnum FROM vmatch WHERE n_match = 1 +), +-- `group_target` — propagate ONE agreed target across a `(unp, annex-number)` chain to members that have no +-- own unique match (value-less admin steps, or later steps whose cumulative value matches no signing_value), +-- but only when the direct members AGREE (a lot-base spread disagrees → no propagation, and the direct hits +-- still stand). So later chain steps link and `current_value` reflects the last step (review MEDIUM 2). +group_target AS ( + SELECT unp, annex_cnum, MIN(resolved_cnum) AS resolved_cnum + FROM direct GROUP BY unp, annex_cnum HAVING COUNT(DISTINCT resolved_cnum) = 1 +) +-- A member links to its OWN unique match if it has one; else to the agreed group target — UNLESS the member +-- is itself value-ambiguous (n_match >= 2), which carries its own contradicting evidence and never links, +-- directly or by inheritance (review todorkolev #2). Members with neither a direct hit nor an agreed group +-- target keep resolved_cnum NULL and stay unlinked. +SELECT g.amendment_id, + COALESCE( + (SELECT d.resolved_cnum FROM direct d WHERE d.amendment_id = g.amendment_id), + (SELECT gt.resolved_cnum FROM group_target gt WHERE gt.unp = g.unp AND gt.annex_cnum = g.annex_cnum) + ) AS resolved_cnum +FROM grp g +WHERE NOT EXISTS ( + SELECT 1 FROM vmatch v WHERE v.amendment_id = g.amendment_id AND v.n_match >= 2 +); + +-- The rewrite UPDATE below correlates raw_amendments.id to amendment_contract_resolve.amendment_id once per +-- unlinked row; index the resolve table so that is a lookup, not a scan (review nikimilenkov LOW 4). +CREATE INDEX IF NOT EXISTS idx_amendment_contract_resolve_id + ON amendment_contract_resolve(amendment_id); + +-- Rewrite contract_number in place — exactly like the #286 УНП bridge — and PRESERVE PROVENANCE (review +-- nikimilenkov MEDIUM 4): keep the original annex-side number in contract_number_raw and stamp link_method +-- so the value-linked rows stay enumerable in staging, through promote, and in the served `amendments` +-- table. contract_number_raw also keeps the annex number in the amendment natural_key (see derive- +-- amendments.sql / promote-amendments.sql), so a resolved row never collides with a native annex that +-- happens to share document_number on the target contract (review nikimilenkov MEDIUM 3). +UPDATE raw_amendments +SET + contract_number_raw = contract_number, + link_method = 'value_anchor', + contract_number = ( + SELECT r.resolved_cnum FROM amendment_contract_resolve r WHERE r.amendment_id = raw_amendments.id + ) +WHERE id IN (SELECT amendment_id FROM amendment_contract_resolve WHERE resolved_cnum IS NOT NULL); + +-- #306 diagnostic (printed by wrangler): annexes linked by the value anchor, and those still unlinked. The +-- two predicates are complementary — both count over the SAME namespace-mismatch population (review +-- nikimilenkov LOW 3) — so linked + still_unlinked = the original mismatch count on a healthy run. +SELECT + (SELECT COUNT(*) FROM raw_amendments WHERE link_method = 'value_anchor') AS annexes_value_linked, + (SELECT COUNT(*) FROM raw_amendments a + WHERE a.source LIKE 'eop:%' AND a.unp IS NOT NULL AND a.contract_number IS NOT NULL + AND a.link_method IS NULL + AND NOT EXISTS (SELECT 1 FROM raw_contracts c + WHERE c.unp = a.unp AND c.contract_number = a.contract_number)) AS eop_annexes_still_unlinked; + +DROP TABLE IF EXISTS amendment_contract_resolve; diff --git a/scripts/work-staging-schema.sql b/scripts/work-staging-schema.sql index 6c403e9c..0e9a581d 100644 --- a/scripts/work-staging-schema.sql +++ b/scripts/work-staging-schema.sql @@ -159,7 +159,9 @@ CREATE TABLE raw_amendments ( fetched_at TEXT NOT NULL, seq_no TEXT, document_number TEXT, - contract_number TEXT, -- ← link to raw_contracts + contract_number TEXT, -- ← link to raw_contracts (rewritten in place by the #306 value resolver) + contract_number_raw TEXT, -- #306: the annex-side number before the value resolver rewrote it (provenance; NULL = never rewritten) + link_method TEXT, -- #306: 'value_anchor' when the resolver linked this row by value; NULL = matched by number / unlinked contract_date TEXT, published_at TEXT, -- amendment publication date (ordering key) unp TEXT, -- ← link to raw_contracts