From fc1878f4d26b50176fcd21e16e0aa110575bc2bc Mon Sep 17 00:00:00 2001 From: Todor Kolev Date: Fri, 17 Jul 2026 02:42:32 +0000 Subject: [PATCH 01/17] fix(etl): enforce bulstat checksum for eik identity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit eik_valid now computes the real Bulstat control digit (9-digit weights 1..8 mod 11 with the remainder-10 reweight, 13-digit second pass), and rejects the degenerate all-zero code. A checksum-invalid ЕИК routes the bidder to a name: key instead of eik:, so falsely-merged distinct companies split apart; a valid ЕИК stays merged correctly. The checksum block is repeated across both ETL paths and pinned identical by a consistency test; fixtures updated to checksum-valid ЕИК-та. Extracts the checksum fix from #203 (@StanislavBG). --- packages/db/src/competition-sql.test.ts | 6 +- .../db/src/eik-checksum-consistency.test.ts | 33 ++++ packages/db/src/queries/companies.test.ts | 10 +- .../src/queries/contracts-filter-sql.test.ts | 10 +- packages/db/src/queries/contracts.test.ts | 4 +- packages/db/src/queries/details.test.ts | 10 +- packages/db/src/queries/identity.test.ts | 2 +- packages/db/src/refresh-slice.test.ts | 128 ++++++++------- packages/db/src/ship-domain.test.ts | 4 +- scripts/normalize-raw.sql | 126 +++++++++++--- scripts/refresh-slice.sql | 154 +++++++++++++++--- 11 files changed, 359 insertions(+), 128 deletions(-) create mode 100644 packages/db/src/eik-checksum-consistency.test.ts diff --git a/packages/db/src/competition-sql.test.ts b/packages/db/src/competition-sql.test.ts index baa0fdb4..20d05aad 100644 --- a/packages/db/src/competition-sql.test.ts +++ b/packages/db/src/competition-sql.test.ts @@ -31,10 +31,10 @@ function readScript(dbPath: string, path: string): void { const FIXTURE = ` INSERT INTO authorities (id, name, bulstat, type_group) VALUES ('auth:A', 'Институция А', '100000001', 'община'), - ('auth:B', 'Институция Б', '100000002', 'община'); + ('auth:B', 'Институция Б', '100000019', 'община'); INSERT INTO bidders (id, name, bulstat, eik_normalized, eik_valid, kind) VALUES - ('eik:X', 'Фирма Х', '200000001', '200000001', 1, 'company'), - ('eik:Y', 'Фирма У', '200000002', '200000002', 1, 'company'); + ('eik:X', 'Фирма Х', '200000002', '200000002', 1, 'company'), + ('eik:Y', 'Фирма У', '200000015', '200000015', 1, 'company'); INSERT INTO tenders (id, source_id, title, authority_id, cpv_code, procedure_type, status) VALUES ('t:A', 'UNP-A', 'Поръчка А', 'auth:A', '45', 'открита процедура', 'awarded'), ('t:B', 'UNP-B', 'Поръчка Б', 'auth:B', '45', 'открита процедура', 'awarded'); diff --git a/packages/db/src/eik-checksum-consistency.test.ts b/packages/db/src/eik-checksum-consistency.test.ts new file mode 100644 index 00000000..606cf4e0 --- /dev/null +++ b/packages/db/src/eik-checksum-consistency.test.ts @@ -0,0 +1,33 @@ +/// +import { readFileSync } from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); + +function checksumBlocks(path: string): string[] { + const sql = readFileSync(path, 'utf8'); + const pattern = + /^[ \t]*CASE\r?\n(?=[ \t]+WHEN eik_clean IS NULL\b[^\r\n]*LENGTH\(eik_clean\) NOT IN \(9, 13\) THEN 0$)[\s\S]*?^[ \t]*END AS eik_valid$/gm; + + return [...sql.matchAll(pattern)].map(([block]) => { + const lines = block.split(/\r?\n/); + const indentation = Math.min( + ...lines.filter((line) => line.trim()).map((line) => line.match(/^[ \t]*/)?.[0].length ?? 0), + ); + return lines.map((line) => line.slice(indentation)).join('\n'); + }); +} + +describe('ЕИК checksum copies stay consistent', () => { + it('keeps every checksum CASE block textually identical', () => { + const normalizeBlocks = checksumBlocks(resolve(root, 'scripts/normalize-raw.sql')); + const refreshBlocks = checksumBlocks(resolve(root, 'scripts/refresh-slice.sql')); + const blocks = [...normalizeBlocks, ...refreshBlocks]; + + expect(normalizeBlocks).toHaveLength(3); + expect(refreshBlocks).toHaveLength(4); + expect(new Set(blocks)).toEqual(new Set([blocks[0]])); + }); +}); diff --git a/packages/db/src/queries/companies.test.ts b/packages/db/src/queries/companies.test.ts index a34c1ad0..a4643d02 100644 --- a/packages/db/src/queries/companies.test.ts +++ b/packages/db/src/queries/companies.test.ts @@ -4,11 +4,11 @@ import type { CompanyTotalsRow } from './rows'; const filteredRows: (CompanyTotalsRow & { sort_value: number })[] = [ { - bidder_id: 'eik:111111111', + bidder_id: 'eik:111111113', name: 'Филтрирана фирма', kind: 'company', ownership_kind: null, - eik: '111111111', + eik: '111111113', eik_valid: 1, settlement: 'София', won_eur: 1000, @@ -25,11 +25,11 @@ const filteredRows: (CompanyTotalsRow & { sort_value: number })[] = [ const unfilteredRows: (CompanyTotalsRow & { sort_value: number })[] = [ ...filteredRows, { - bidder_id: 'eik:999999999', + bidder_id: 'eik:999999995', name: 'Нефилтрирана фирма', kind: 'company', ownership_kind: null, - eik: '999999999', + eik: '999999995', eik_valid: 1, settlement: 'Пловдив', won_eur: 900, @@ -93,7 +93,7 @@ describe('streamCompaniesCsv', () => { .map((line) => line.split(',')[0]); expect(csvEiks).toEqual(list.items.map((item) => item.eik)); - expect(csvEiks).toEqual(['111111111']); + expect(csvEiks).toEqual(['111111113']); }); it('exports fewer rows for a sector filter than for the unfiltered corpus', async () => { diff --git a/packages/db/src/queries/contracts-filter-sql.test.ts b/packages/db/src/queries/contracts-filter-sql.test.ts index 2c4d6974..98314206 100644 --- a/packages/db/src/queries/contracts-filter-sql.test.ts +++ b/packages/db/src/queries/contracts-filter-sql.test.ts @@ -26,14 +26,14 @@ const FIXTURE = ` INSERT INTO authorities (id, name, bulstat, type_group) VALUES ('auth:100000001', 'Институция А', '100000001', 'община'); INSERT INTO bidders (id, name, bulstat, eik_normalized, eik_valid, kind) VALUES - ('eik:200000001', 'Фирма Х', '200000001', '200000001', 1, 'company'); + ('eik:200000002', 'Фирма Х', '200000002', '200000002', 1, 'company'); INSERT INTO tenders (id, source_id, title, authority_id, cpv_code, procedure_type, status) VALUES ('t:A', 'UNP-A', 'Поръчка А', 'auth:100000001', '45000000', 'открита процедура', 'awarded'); INSERT INTO contracts (id, tender_id, bidder_id, amount, currency, signed_at, bids_received, value_flag, amount_eur) VALUES - ('c:NULL', 't:A', 'eik:200000001', 100, 'EUR', '2024-01-01', NULL, 'ok', 100), - ('c:ZERO', 't:A', 'eik:200000001', 200, 'EUR', '2024-01-02', 0, 'ok', 200), - ('c:ONE', 't:A', 'eik:200000001', 300, 'EUR', '2024-01-03', 1, 'ok', 300), - ('c:TWO', 't:A', 'eik:200000001', 400, 'EUR', '2024-01-04', 2, 'ok', 400); + ('c:NULL', 't:A', 'eik:200000002', 100, 'EUR', '2024-01-01', NULL, 'ok', 100), + ('c:ZERO', 't:A', 'eik:200000002', 200, 'EUR', '2024-01-02', 0, 'ok', 200), + ('c:ONE', 't:A', 'eik:200000002', 300, 'EUR', '2024-01-03', 1, 'ok', 300), + ('c:TWO', 't:A', 'eik:200000002', 400, 'EUR', '2024-01-04', 2, 'ok', 400); `; /** Minimal D1Database facade over node:sqlite — enough for the query layer's prepare/bind/all/first. */ diff --git a/packages/db/src/queries/contracts.test.ts b/packages/db/src/queries/contracts.test.ts index 504754b2..d2339111 100644 --- a/packages/db/src/queries/contracts.test.ts +++ b/packages/db/src/queries/contracts.test.ts @@ -21,9 +21,9 @@ const contractRow = { unp: 'UNP-1', cpv_code: '45000000', eu_funded: 0, - authority_id: 'auth:123456789', + authority_id: 'auth:123456786', authority_name: 'Authority', - bidder_id: 'eik:111111111', + bidder_id: 'eik:111111113', bidder_name: 'Bidder', bidder_kind: 'company' as const, procedure_type: 'Открита процедура', diff --git a/packages/db/src/queries/details.test.ts b/packages/db/src/queries/details.test.ts index 04e3cd00..ffe35cff 100644 --- a/packages/db/src/queries/details.test.ts +++ b/packages/db/src/queries/details.test.ts @@ -41,14 +41,14 @@ const baseContractRow = { tender_fx_rate: null as number | null, start_date: null, end_date: null, - authority_id: 'auth:123456789', + authority_id: 'auth:123456786', authority_name: 'Authority', authority_type_group: 'ministry', authority_settlement: 'Sofia', - bidder_id: 'eik:111111111', + bidder_id: 'eik:111111113', bidder_name: 'Bidder', bidder_kind: 'company' as const, - bidder_eik: '111111111', + bidder_eik: '111111113', bidder_settlement: 'Sofia', }; @@ -102,7 +102,7 @@ describe('getContract', () => { estimated_fx_rate: null, bidder_name: 'Bidder', bidder_kind: 'company', - bidder_id: 'eik:111111111', + bidder_id: 'eik:111111113', }, { lot_id: 'lot:UNP-1:2', @@ -147,7 +147,7 @@ describe('getContract', () => { estimated_fx_rate: 0.9, bidder_name: 'Bidder', bidder_kind: 'company', - bidder_id: 'eik:111111111', + bidder_id: 'eik:111111113', }, { lot_id: 'lot:UNP-1:2', diff --git a/packages/db/src/queries/identity.test.ts b/packages/db/src/queries/identity.test.ts index 0cdc4272..2acc6ccf 100644 --- a/packages/db/src/queries/identity.test.ts +++ b/packages/db/src/queries/identity.test.ts @@ -23,7 +23,7 @@ describe('company slug', () => { expect(bidderIdFromSlug(slug)).toBe(id); }); it('round-trips a 13-digit ЕИК', () => { - expect(bidderIdFromSlug(companySlug('eik:1234567890123'))).toBe('eik:1234567890123'); + expect(bidderIdFromSlug(companySlug('eik:8316417910124'))).toBe('eik:8316417910124'); }); }); diff --git a/packages/db/src/refresh-slice.test.ts b/packages/db/src/refresh-slice.test.ts index aa20ec76..3318424c 100644 --- a/packages/db/src/refresh-slice.test.ts +++ b/packages/db/src/refresh-slice.test.ts @@ -64,11 +64,11 @@ INSERT INTO raw_tenders VALUES ('eop:tenders:2026-06-01', 2026, '2026-06-07T00:00:00Z', 'UNP-CE-1', 'TENDER-CE-1', 'open', 'Base tender', '45000000', 'Construction', 'works', 2000, 'BGN', 'basis', - 'lowest', 'Authority CE', '123456789', 'public', 'activity', '2026-06-10', 'notice', + 'lowest', 'Authority CE', '123456786', 'public', 'activity', '2026-06-10', 'notice', NULL, NULL, 1, 0, '2026-06-01'), ('eop:tenders:2026-06-01', 2026, '2026-06-07T00:00:00Z', 'UNP-CE-1', 'TENDER-CE-1', 'open', 'Base tender', '45000000', 'Construction', 'works', 2000, 'BGN', 'basis', - 'lowest', 'Authority CE', '123456789', 'public', 'activity', '2026-06-10', 'notice', + 'lowest', 'Authority CE', '123456786', 'public', 'activity', '2026-06-10', 'notice', '1', 'Lot 1', 1, 0, '2026-06-01'); INSERT INTO raw_contracts @@ -82,12 +82,12 @@ INSERT INTO raw_contracts VALUES ('eop:contracts:2026-06-01', 2026, 'eop', '2026-06-07T00:00:00Z', 0, 'DOC-CE-1', '2026-06-01', 'UNP-CE-1', 'TENDER-CE-1', 'open', 'Base tender', '45000000', - 'Construction', 'works', 2000, 'BGN', 'basis', 'lowest', 'Authority CE', '123456789', + 'Construction', 'works', 2000, 'BGN', 'basis', 'lowest', 'Authority CE', '123456786', 'public', 'activity', 'notice', '1', 'CONTRACT-CE-1', '2026-06-02', 1000, 'BGN', - 'Base contract', 0, '987654321', 'Bidder CE', 'BG', 'small', 0, 3, 1, 0, 0, 30), + 'Base contract', 0, '987654308', 'Bidder CE', 'BG', 'small', 0, 3, 1, 0, 0, 30), ('ocds:2026-06-01', 2026, 'ocds', '2026-06-07T00:00:00Z', 0, 'DOC-CO-1', '2026-06-01', 'OCDS-CO-1', 'TENDER-CO-1', 'open', 'OCDS tender', '45000000', - 'Construction', 'works', 5000, 'BGN', 'basis', 'lowest', 'Authority CE', '123456789', + 'Construction', 'works', 5000, 'BGN', 'basis', 'lowest', 'Authority CE', '123456786', 'public', 'activity', 'notice', NULL, 'CONTRACT-CO-1', '2026-06-02', 1000, 'BGN', 'OCDS contract', 0, '987654322', 'Bidder CO', 'BG', 'small', 0, 3, 1, 0, 0, 30); @@ -98,10 +98,10 @@ INSERT INTO raw_amendments currency, description) VALUES ('eop:annexes:2026-06-01', 2026, 'eop', '2026-06-07T00:00:00Z', '1', 'AMD-CE-1', - 'CONTRACT-CE-1', '2026-06-02', '2026-06-03', 'UNP-CE-1', '123456789', 'Authority CE', + 'CONTRACT-CE-1', '2026-06-02', '2026-06-03', 'UNP-CE-1', '123456786', 'Authority CE', 'Base tender', 'works', 1000, 1200, 200, 'BGN', 'Increase'), ('ocds:2026-06-01', 2026, 'ocds', '2026-06-07T00:00:00Z', '1', 'AMD-CO-1', - 'CONTRACT-CO-1', '2026-06-02', '2026-06-03', 'OCDS-CO-1', '123456789', 'Authority CE', + 'CONTRACT-CO-1', '2026-06-02', '2026-06-03', 'OCDS-CO-1', '123456786', 'Authority CE', 'OCDS tender', 'works', 1000, 1300, 300, 'BGN', 'OCDS increase'); `, ); @@ -117,7 +117,7 @@ function seedRepeatedAnnexOnly(dbPath: string): void { currency, description) VALUES ('eop:annexes:2026-06-02', 2026, 'eop', '2026-06-08T00:00:00Z', '1', 'AMD-CE-1', - 'CONTRACT-CE-1', '2026-06-02', '2026-06-03', 'UNP-CE-1', '123456789', 'Authority CE', + 'CONTRACT-CE-1', '2026-06-02', '2026-06-03', 'UNP-CE-1', '123456786', 'Authority CE', 'Base tender', 'works', 1000, 1200, 200, 'BGN', 'Increase');`, ); } @@ -133,7 +133,7 @@ function seedEopOnlySharedNumber(dbPath: string): void { VALUES ('eop:tenders:2026-06-01', 2026, '2026-06-07T00:00:00Z', 'UNP-SHARED', 'TENDER-SHARED', 'open', 'Shared tender', '45000000', 'Construction', 'works', 5000, 'BGN', 'basis', - 'lowest', 'Authority Shared', '223456789', 'public', 'activity', '2026-06-10', 'notice', + 'lowest', 'Authority Shared', '223456787', 'public', 'activity', '2026-06-10', 'notice', NULL, NULL, 1, 0, '2026-06-01'); INSERT INTO raw_contracts @@ -147,7 +147,7 @@ function seedEopOnlySharedNumber(dbPath: string): void { VALUES ('eop:contracts:2026-06-01', 2026, 'eop', '2026-06-07T00:00:00Z', 0, 'DOC-SHARED', '2026-06-01', 'UNP-SHARED', 'TENDER-SHARED', 'open', 'Shared tender', '45000000', - 'Construction', 'works', 5000, 'BGN', 'basis', 'lowest', 'Authority Shared', '223456789', + 'Construction', 'works', 5000, 'BGN', 'basis', 'lowest', 'Authority Shared', '223456787', 'public', 'activity', 'notice', NULL, 'CONTRACT-SHARED', '2026-06-02', 1000, 'BGN', 'Shared contract', 0, '887654321', 'Bidder Shared', 'BG', 'small', 0, 3, 1, 0, 0, 30);`, ); @@ -167,7 +167,7 @@ function seedOcdsOnlySharedNumber(dbPath: string): void { VALUES ('ocds:2026-06-02', 2026, 'ocds', '2026-06-08T00:00:00Z', 0, 'DOC-SHARED-O', '2026-06-02', 'OCDS-SHARED', 'TENDER-SHARED-O', 'open', 'Shared tender ocds', '45000000', - 'Construction', 'works', 5000, 'BGN', 'basis', 'lowest', 'Authority Shared', '223456789', + 'Construction', 'works', 5000, 'BGN', 'basis', 'lowest', 'Authority Shared', '223456787', 'public', 'activity', 'notice', NULL, 'CONTRACT-SHARED', '2026-06-02', 1000, 'BGN', 'Shared contract ocds', 0, '887654321', 'Bidder Shared', 'BG', 'small', 0, 3, 1, 0, 0, 30);`, ); @@ -189,11 +189,11 @@ function seedContractIdFixture(dbPath: string): void { VALUES ('eop:tenders:2026-06-01', 2026, '2026-06-07T00:00:00Z', 'UNP-ID', 'TENDER-ID', 'open', 'ID tender', '45000000', 'Construction', 'works', 10000, 'BGN', 'basis', - 'lowest', 'Authority ID', '523456789', 'public', 'activity', '2026-06-10', 'notice', + 'lowest', 'Authority ID', '523456782', 'public', 'activity', '2026-06-10', 'notice', NULL, NULL, 1, 0, '2026-06-01'), ('eop:tenders:2026-06-01', 2026, '2026-06-07T00:00:00Z', 'UNP-ID', 'TENDER-ID', 'open', 'ID tender', '45000000', 'Construction', 'works', 10000, 'BGN', 'basis', - 'lowest', 'Authority ID', '523456789', 'public', 'activity', '2026-06-10', 'notice', + 'lowest', 'Authority ID', '523456782', 'public', 'activity', '2026-06-10', 'notice', '1', 'Lot 1', 1, 0, '2026-06-01'); INSERT INTO raw_contracts @@ -207,29 +207,29 @@ function seedContractIdFixture(dbPath: string): void { VALUES ('eop:contracts:2026-06-01', 2026, 'eop', '2026-06-07T00:00:00Z', 0, 'DOC-OLD', '2026-06-01', 'UNP-ID', 'TENDER-ID', 'open', 'ID tender', '45000000', - 'Construction', 'works', 10000, 'BGN', 'basis', 'lowest', 'Authority ID', '523456789', + 'Construction', 'works', 10000, 'BGN', 'basis', 'lowest', 'Authority ID', '523456782', 'public', 'activity', 'notice', '1', 'CONTRACT-ID', '2026-06-02', 1000, 'BGN', - 'Old duplicate', 0, '577777777', 'Bidder ID', 'BG', 'small', 0, 3, 1, 0, 0, 30), + 'Old duplicate', 0, '577777760', 'Bidder ID', 'BG', 'small', 0, 3, 1, 0, 0, 30), ('eop:contracts:2026-06-02', 2026, 'eop', '2026-06-08T00:00:00Z', 0, 'DOC-NEW', '2026-06-02', 'UNP-ID', 'TENDER-ID', 'open', 'ID tender', '45000000', - 'Construction', 'works', 10000, 'BGN', 'basis', 'lowest', 'Authority ID', '523456789', + 'Construction', 'works', 10000, 'BGN', 'basis', 'lowest', 'Authority ID', '523456782', 'public', 'activity', 'notice', '1', 'CONTRACT-ID', '2026-06-02', 1000, 'BGN', - 'New duplicate', 0, '577777777', 'Bidder ID', 'BG', 'small', 0, 3, 1, 0, 0, 30), + 'New duplicate', 0, '577777760', 'Bidder ID', 'BG', 'small', 0, 3, 1, 0, 0, 30), ('eop:contracts:2026-06-02', 2026, 'eop', '2026-06-08T00:00:00Z', 0, 'DOC-NULL', '2026-06-02', 'UNP-ID', 'TENDER-ID', 'open', 'ID tender', '45000000', - 'Construction', 'works', 10000, 'BGN', 'basis', 'lowest', 'Authority ID', '523456789', + 'Construction', 'works', 10000, 'BGN', 'basis', 'lowest', 'Authority ID', '523456782', 'public', 'activity', 'notice', NULL, NULL, '2026-06-03', 500, 'BGN', 'Null number', 0, '577777778', 'Bidder Null', 'BG', 'small', 0, 1, 1, 0, 0, 30), ('ocds:2026-06-02', 2026, 'ocds', '2026-06-08T00:00:00Z', 0, 'DOC-OCDS', '2026-06-02', 'OCDS-ID', 'TENDER-OCDS', 'open', 'OCDS ID tender', '45000000', - 'Construction', 'works', 10000, 'BGN', 'basis', 'lowest', 'Authority ID', '523456789', + 'Construction', 'works', 10000, 'BGN', 'basis', 'lowest', 'Authority ID', '523456782', 'public', 'activity', 'notice', 'LOT-0001', 'CONTRACT-OCDS', '2026-06-04', 700, 'BGN', - 'OCDS ID', 0, '577777779', 'Bidder OCDS', 'BG', 'small', 0, 2, 1, 0, 0, 30), + 'OCDS ID', 0, '577777792', 'Bidder OCDS', 'BG', 'small', 0, 2, 1, 0, 0, 30), ('ocds:2026-06-02', 2026, 'ocds', '2026-06-08T00:00:00Z', 0, 'DOC-OCDS-SKIP', '2026-06-02', 'OCDS-ID-SKIP', 'TENDER-OCDS-SKIP', 'open', 'OCDS skip tender', '45000000', - 'Construction', 'works', 10000, 'BGN', 'basis', 'lowest', 'Authority ID', '523456789', + 'Construction', 'works', 10000, 'BGN', 'basis', 'lowest', 'Authority ID', '523456782', 'public', 'activity', 'notice', NULL, 'CONTRACT-ID', '2026-06-04', 700, 'BGN', - 'OCDS skip', 0, '577777777', 'Bidder ID', 'BG', 'small', 0, 2, 1, 0, 0, 30);`, + 'OCDS skip', 0, '577777760', 'Bidder ID', 'BG', 'small', 0, 2, 1, 0, 0, 30);`, ); } @@ -259,7 +259,7 @@ function seedSyntheticWindow( '2026-06-02', ${sqlValue(values.unp)}, ${sqlValue(`TENDER-${values.unp}`)}, 'open', ${sqlValue(values.subject)}, ${sqlValue(values.cpv)}, 'Construction', 'works', ${sqlValue(values.estimated)}, ${sqlValue(values.currency)}, - 'basis', 'lowest', 'Authority Synthetic', '623456789', 'public', 'activity', 'notice', + 'basis', 'lowest', 'Authority Synthetic', '623456780', 'public', 'activity', 'notice', NULL, ${sqlValue(`CONTRACT-${values.source}`)}, '2026-06-02', 100, ${sqlValue(values.currency)}, 'Synthetic contract', 0, '667777777', 'Bidder Synthetic', 'BG', 'small', 0, 1, 1, 0, 0, 30);`, ); @@ -269,7 +269,7 @@ function seedSyntheticAuthority(dbPath: string): void { sqlite( dbPath, `INSERT OR IGNORE INTO authorities (id, name, bulstat, type, type_group) - VALUES ('auth:623456789', 'Authority Synthetic', '623456789', 'public', 'other');`, + VALUES ('auth:623456780', 'Authority Synthetic', '623456780', 'public', 'other');`, ); } @@ -281,7 +281,7 @@ function seedReplayParty(dbPath: string): void { region_nuts, contact_email, contact_phone) VALUES ('ocds:2026-06-01', '2026-06-08T00:00:00Z', 'ocds-replay', 'party-replay', - '823456789', 'Replay party', 'Replay street 1', 'Replay city', 'BG411', + '823456782', 'Replay party', 'Replay street 1', 'Replay city', 'BG411', 'replay@example.test', '+359000000001'), ('ocds:2026-06-01', '2026-06-08T00:00:00Z', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL);`, @@ -302,9 +302,9 @@ function seedReplayEntity(dbPath: string): void { VALUES ('eop:contracts:2026-06-02', 2026, 'eop', '2026-06-08T00:00:00Z', 0, 'DOC-REPLAY', '2026-06-02', 'UNP-REPLAY', 'TENDER-REPLAY', 'open', 'Replay tender', '45000000', - 'Construction', 'works', 1000, 'BGN', 'basis', 'lowest', 'Replay authority', '823456789', + 'Construction', 'works', 1000, 'BGN', 'basis', 'lowest', 'Replay authority', '823456782', 'public', 'activity', 'notice', NULL, 'CONTRACT-REPLAY', '2026-06-02', 100, 'BGN', - 'Replay contract', 0, '823456789', 'Replay bidder', 'BG', 'small', 0, 1, 1, 0, 0, 30);`, + 'Replay contract', 0, '823456782', 'Replay bidder', 'BG', 'small', 0, 1, 1, 0, 0, 30);`, ); } @@ -320,16 +320,16 @@ function seedCollidingPartySlot(dbPath: string): void { region_nuts, contact_email, contact_phone) VALUES ('ocds:2026-06-01', '2026-06-08T00:00:00Z', 'ocds-shared', 'ORG-0003', - '111111111', 'Company A', 'Addr A 1', 'City A', 'BG331', 'a@example.test', '+359000000011'), + '111111113', 'Company A', 'Addr A 1', 'City A', 'BG331', 'a@example.test', '+359000000011'), ('ocds:2026-06-05', '2026-06-08T00:00:00Z', 'ocds-shared', 'ORG-0003', - '222222222', 'Company B', 'Addr B 2', 'City B', 'BG421', 'b@example.test', '+359000000022');`, + '222222226', 'Company B', 'Addr B 2', 'City B', 'BG421', 'b@example.test', '+359000000022');`, ); } function seedTwoCollidingBidders(dbPath: string): void { for (const [eik, name] of [ - ['111111111', 'Company A'], - ['222222222', 'Company B'], + ['111111113', 'Company A'], + ['222222226', 'Company B'], ] as const) { sqlite( dbPath, @@ -345,7 +345,7 @@ function seedTwoCollidingBidders(dbPath: string): void { (${sqlValue(`eop:contracts:2026-06-02:${eik}`)}, 2026, 'eop', '2026-06-08T00:00:00Z', 0, ${sqlValue(`DOC-${eik}`)}, '2026-06-02', ${sqlValue(`UNP-${eik}`)}, ${sqlValue(`TENDER-${eik}`)}, 'open', 'Slot tender', '45000000', 'Construction', 'works', 1000, 'BGN', 'basis', 'lowest', - 'Slot authority', '923456789', 'public', 'activity', 'notice', NULL, + 'Slot authority', '923456783', 'public', 'activity', 'notice', NULL, ${sqlValue(`CONTRACT-${eik}`)}, '2026-06-02', 100, 'BGN', 'Slot contract', 0, ${sqlValue(eik)}, ${sqlValue(name)}, 'BG', 'small', 0, 1, 1, 0, 0, 30);`, ); @@ -488,15 +488,15 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, workStagingSchemaPath); sqlite( dbPath, - `INSERT INTO authorities (id, name, bulstat, type) VALUES ('auth:323456789', 'Authority Flag', '323456789', 'public'); - INSERT INTO bidders (id, name, bulstat, eik_normalized, eik_valid, kind) VALUES ('eik:777777777', 'Bidder Flag', '777777777', '777777777', 1, 'company'); + `INSERT INTO authorities (id, name, bulstat, type) VALUES ('auth:323456788', 'Authority Flag', '323456788', 'public'); + INSERT INTO bidders (id, name, bulstat, eik_normalized, eik_valid, kind) VALUES ('eik:777777771', 'Bidder Flag', '777777771', '777777771', 1, 'company'); INSERT INTO tenders (id, source_id, title, authority_id, estimated_value, currency, procedure_type, status) - VALUES ('t:UNP-FLAG', 'UNP-FLAG', 'Flag tender', 'auth:323456789', 1000, 'BGN', 'open', 'awarded'); + VALUES ('t:UNP-FLAG', 'UNP-FLAG', 'Flag tender', 'auth:323456788', 1000, 'BGN', 'open', 'awarded'); INSERT INTO contracts (id, tender_id, bidder_id, amount, currency, signed_at, contract_number, signing_value, current_value, annex_count, value_flag, amount_eur, signing_value_eur) VALUES - ('c:e:flag', 't:UNP-FLAG', 'eik:777777777', 100, 'BGN', '2026-06-02', + ('c:e:flag', 't:UNP-FLAG', 'eik:777777771', 100, 'BGN', '2026-06-02', 'CONTRACT-FLAG', 100, 10000, 1, 'annex_suspect', NULL, 100 / 1.95583); INSERT INTO raw_amendments (source, dataset_year, dataset_variant, fetched_at, seq_no, document_number, @@ -505,7 +505,7 @@ describe('refresh-slice EOP base derivation', () => { currency, description) VALUES ('eop:annexes:2026-06-02', 2026, 'eop', '2026-06-08T00:00:00Z', '2', 'AMD-FLAG-2', - 'CONTRACT-FLAG', '2026-06-02', '2026-06-03', 'UNP-FLAG', '323456789', 'Authority Flag', + 'CONTRACT-FLAG', '2026-06-02', '2026-06-03', 'UNP-FLAG', '323456788', 'Authority Flag', 'Flag tender', 'works', 10000, 120, -9880, 'BGN', 'Normalize');`, ); @@ -560,7 +560,7 @@ describe('refresh-slice EOP base derivation', () => { '2026-06-01', 'UNP-MISMATCH', 'TENDER-MISMATCH', 'open', 'Mismatch tender', '45000000', 'Construction', 'works', 1000, 'BGN', 'basis', 'lowest', 'Authority Mismatch', '423456789', 'public', 'activity', 'notice', NULL, 'CONTRACT-MISMATCH', '2026-06-02', 1000, 'BGN', - 'Mismatch contract', 0, '677777777', 'Bidder Mismatch', 'BG', 'small', 0, 3, 1, 0, 0, 30); + 'Mismatch contract', 0, '677777779', 'Bidder Mismatch', 'BG', 'small', 0, 3, 1, 0, 0, 30); INSERT INTO raw_amendments (source, dataset_year, dataset_variant, fetched_at, seq_no, document_number, @@ -660,7 +660,7 @@ describe('refresh-slice EOP base derivation', () => { VALUES ('eop:tenders:2026-06-01', 2026, '2026-06-07T00:00:00Z', 'UNP-ORD', 'TENDER-ORD', 'open', 'Ordinal tender', '45000000', 'Construction', 'works', 10000, 'BGN', 'basis', - 'lowest', 'Authority Ordinal', '723456789', 'public', 'activity', '2026-06-10', 'notice', + 'lowest', 'Authority Ordinal', '723456781', 'public', 'activity', '2026-06-10', 'notice', NULL, NULL, 1, 0, '2026-06-01'); INSERT INTO raw_amendments @@ -670,7 +670,7 @@ describe('refresh-slice EOP base derivation', () => { currency, description) VALUES ('eop:annexes:2026-06-01', 2026, 'eop', '2026-06-07T00:00:00Z', '1', 'AMD-ORD', - 'CONTRACT-ORD', '2026-06-02', '2026-06-03', 'UNP-ORD', '723456789', 'Authority Ordinal', + 'CONTRACT-ORD', '2026-06-02', '2026-06-03', 'UNP-ORD', '723456781', 'Authority Ordinal', 'Ordinal tender', 'works', 100, 150, 50, 'BGN', 'Increase');`, ); } @@ -687,12 +687,12 @@ describe('refresh-slice EOP base derivation', () => { VALUES ('eop:contracts:2026-06-01', 2026, 'eop', '2026-06-07T00:00:00Z', 0, 'DOC-A', '2026-06-01', 'UNP-ORD', 'TENDER-ORD', 'open', 'Ordinal tender', '45000000', - 'Construction', 'works', 10000, 'BGN', 'basis', 'lowest', 'Authority Ordinal', '723456789', + 'Construction', 'works', 10000, 'BGN', 'basis', 'lowest', 'Authority Ordinal', '723456781', 'public', 'activity', 'notice', NULL, 'CONTRACT-ORD', '2026-06-02', 100, 150, 1, 'BGN', 'Ordinal A', 0, '777777771', 'Bidder Ordinal', 'BG', 'small', 0, 1, 1, 0, 0, 30), ('eop:contracts:2026-06-01', 2026, 'eop', '2026-06-07T00:00:00Z', 0, 'DOC-B', '2026-06-01', 'UNP-ORD', 'TENDER-ORD', 'open', 'Ordinal tender', '45000000', - 'Construction', 'works', 10000, 'BGN', 'basis', 'lowest', 'Authority Ordinal', '723456789', + 'Construction', 'works', 10000, 'BGN', 'basis', 'lowest', 'Authority Ordinal', '723456781', 'public', 'activity', 'notice', NULL, 'CONTRACT-ORD', '2026-06-02', 100, 150, 1, 'BGN', 'Ordinal B', 0, '777777771', 'Bidder Ordinal', 'BG', 'small', 0, 1, 1, 0, 0, 30);`, ); @@ -709,12 +709,12 @@ describe('refresh-slice EOP base derivation', () => { VALUES ('eop:contracts:2026-06-01', 2026, 'eop', '2026-06-07T00:00:00Z', 0, 'DOC-A', '2026-06-01', 'UNP-ORD', 'TENDER-ORD', 'open', 'Ordinal tender', '45000000', - 'Construction', 'works', 10000, 'BGN', 'basis', 'lowest', 'Authority Ordinal', '723456789', + 'Construction', 'works', 10000, 'BGN', 'basis', 'lowest', 'Authority Ordinal', '723456781', 'public', 'activity', 'notice', NULL, 'CONTRACT-ORD', '2026-06-02', 100, NULL, 0, 'BGN', 'Ordinal A', 0, '777777771', 'Bidder Ordinal', 'BG', 'small', 0, 1, 1, 0, 0, 30), ('eop:contracts:2026-06-01', 2026, 'eop', '2026-06-07T00:00:00Z', 0, 'DOC-B', '2026-06-01', 'UNP-ORD', 'TENDER-ORD', 'open', 'Ordinal tender', '45000000', - 'Construction', 'works', 10000, 'BGN', 'basis', 'lowest', 'Authority Ordinal', '723456789', + 'Construction', 'works', 10000, 'BGN', 'basis', 'lowest', 'Authority Ordinal', '723456781', 'public', 'activity', 'notice', NULL, 'CONTRACT-ORD', '2026-06-02', 100, NULL, 0, 'BGN', 'Ordinal B', 0, '777777771', 'Bidder Ordinal', 'BG', 'small', 0, 1, 1, 0, 0, 30);`, ); @@ -833,7 +833,7 @@ describe('refresh-slice EOP base derivation', () => { VALUES ('eop:tenders:2026-06-01', 2026, '2026-06-07T00:00:00Z', 'UNP-REAL', 'TENDER-REAL', NULL, 'Real header', '99000000', 'Header', 'works', 990, 'EUR', 'basis', - 'lowest', 'Authority Synthetic', '623456789', 'public', 'activity', '2026-06-10', 'notice', + 'lowest', 'Authority Synthetic', '623456780', 'public', 'activity', '2026-06-10', 'notice', NULL, NULL, 1, 0, '2026-06-01');`, ); seedSyntheticWindow(dbPath, { @@ -884,11 +884,17 @@ describe('refresh-slice EOP base derivation', () => { readScript(sliceDb, refreshSlicePath); const authoritySql = - "SELECT id, nuts, settlement, address, contact_email, contact_phone FROM authorities WHERE id='auth:823456789'"; + "SELECT id, nuts, settlement, address, contact_email, contact_phone FROM authorities WHERE id='auth:823456782'"; const bidderSql = - "SELECT id, nuts, settlement, address, contact_email, contact_phone FROM bidders WHERE id='eik:823456789'"; - expect(sqliteJson(fullDb, authoritySql)).toEqual(sqliteJson(sliceDb, authoritySql)); - expect(sqliteJson(fullDb, bidderSql)).toEqual(sqliteJson(sliceDb, bidderSql)); + "SELECT id, nuts, settlement, address, contact_email, contact_phone FROM bidders WHERE id='eik:823456782'"; + const fullAuthorities = sqliteJson(fullDb, authoritySql); + const sliceAuthorities = sqliteJson(sliceDb, authoritySql); + const fullBidders = sqliteJson(fullDb, bidderSql); + const sliceBidders = sqliteJson(sliceDb, bidderSql); + expect(fullAuthorities).not.toHaveLength(0); + expect(sliceAuthorities).toEqual(fullAuthorities); + expect(fullBidders).not.toHaveLength(0); + expect(sliceBidders).toEqual(fullBidders); const partiesAfterWindow2 = sqliteJson(sliceDb, 'SELECT * FROM parties ORDER BY party_key'); const authorityAfterWindow2 = sqliteJson(sliceDb, authoritySql); @@ -920,19 +926,19 @@ describe('refresh-slice EOP base derivation', () => { const sliceDb = resolve(dir, 'slice.sqlite'); try { const partyByEik = - "SELECT eik, locality, street_address, contact_email FROM parties WHERE eik IN ('111111111','222222222') ORDER BY eik"; + "SELECT eik, locality, street_address, contact_email FROM parties WHERE eik IN ('111111113','222222226') ORDER BY eik"; const bidderByEik = - "SELECT eik_normalized AS eik, settlement, address, contact_email FROM bidders WHERE eik_normalized IN ('111111111','222222222') ORDER BY eik_normalized"; + "SELECT eik_normalized AS eik, settlement, address, contact_email FROM bidders WHERE eik_normalized IN ('111111113','222222226') ORDER BY eik_normalized"; // Each company must be enriched from its OWN party row, not the latest slot occupant's. const expectedParties = [ { - eik: '111111111', + eik: '111111113', locality: 'City A', street_address: 'Addr A 1', contact_email: 'a@example.test', }, { - eik: '222222222', + eik: '222222226', locality: 'City B', street_address: 'Addr B 2', contact_email: 'b@example.test', @@ -940,13 +946,13 @@ describe('refresh-slice EOP base derivation', () => { ]; const expectedBidders = [ { - eik: '111111111', + eik: '111111113', settlement: 'City A', address: 'Addr A 1', contact_email: 'a@example.test', }, { - eik: '222222222', + eik: '222222226', settlement: 'City B', address: 'Addr B 2', contact_email: 'b@example.test', @@ -988,7 +994,7 @@ function seedReattrContract(dbPath: string, source: string, eik: string, name: s VALUES (${sqlValue(source)}, 2026, 'eop', '2026-06-08T00:00:00Z', 0, 'DOC-REATTR', '2026-06-02', 'UNP-REATTR', 'TENDER-REATTR', 'open', 'Reattr tender', '45000000', - 'Construction', 'works', 1000, 'BGN', 'basis', 'lowest', 'Reattr authority', '923456789', + 'Construction', 'works', 1000, 'BGN', 'basis', 'lowest', 'Reattr authority', '923456783', 'public', 'activity', 'notice', NULL, 'CONTRACT-REATTR', '2026-06-02', 100, 'BGN', 'Reattr contract', 0, ${sqlValue(eik)}, ${sqlValue(name)}, 'BG', 'small', 0, 1, 1, 0, 0, 30);`, ); @@ -1012,18 +1018,18 @@ describe('refresh-slice integrity gate', () => { try { initWorkDb(dbPath); // window 1: contract attributed to bidder A - seedReattrContract(dbPath, 'eop:contracts:2026-06-02', '111111111', 'Company A'); + seedReattrContract(dbPath, 'eop:contracts:2026-06-02', '111111113', 'Company A'); readScript(dbPath, refreshSlicePath); - expect(wonEur('111111111')).toBeGreaterThan(0); + expect(wonEur('111111113')).toBeGreaterThan(0); // window 2: same contract_number+unp re-imported with a different contractor → re-attribution resetRawStaging(dbPath); - seedReattrContract(dbPath, 'eop:contracts:2026-06-05', '222222222', 'Company B'); + seedReattrContract(dbPath, 'eop:contracts:2026-06-05', '222222226', 'Company B'); readScript(dbPath, refreshSlicePath); // A's scoped rollup was rebuilt to empty (the value moved), B now carries it… - expect(wonEur('111111111')).toBe(0); - expect(wonEur('222222222')).toBeGreaterThan(0); + expect(wonEur('111111113')).toBe(0); + expect(wonEur('222222226')).toBeGreaterThan(0); // …so the gate reconciles on the slice-built DB (rollup check runs; staging self-skips here). const results = assertIntegrity(run, { label: 'test-slice', exit: false }); diff --git a/packages/db/src/ship-domain.test.ts b/packages/db/src/ship-domain.test.ts index e1249ed7..7101ec83 100644 --- a/packages/db/src/ship-domain.test.ts +++ b/packages/db/src/ship-domain.test.ts @@ -49,9 +49,9 @@ describe('ship-domain', () => { workDb, `INSERT INTO authorities (id, name, bulstat, type) VALUES ('auth:1', 'Authority line 1 Authority line 2', '1', 'public'); - INSERT INTO bidders (id, name, bulstat, eik_normalized, eik_valid, kind) VALUES ('eik:200000007', 'Bidder', '200000007', '200000007', 1, 'company'); + INSERT INTO bidders (id, name, bulstat, eik_normalized, eik_valid, kind) VALUES ('eik:200000002', 'Bidder', '200000002', '200000002', 1, 'company'); INSERT INTO tenders (id, source_id, title, authority_id, currency, procedure_type, status) VALUES ('t:1', '1', 'Tender', 'auth:1', 'BGN', 'open', 'awarded'); - INSERT INTO contracts (id, tender_id, bidder_id, amount, currency, contract_number, signing_value, value_flag, amount_eur) VALUES ('c:e:1', 't:1', 'eik:200000007', 10, 'BGN', 'C1', 10, 'ok', 10 / 1.95583); + INSERT INTO contracts (id, tender_id, bidder_id, amount, currency, contract_number, signing_value, value_flag, amount_eur) VALUES ('c:e:1', 't:1', 'eik:200000002', 10, 'BGN', 'C1', 10, 'ok', 10 / 1.95583); INSERT INTO amendments (id, natural_key, contract_number, unp, description, source) VALUES ('am:1:C1:A1', 'am:1:C1:A1', 'C1', '1', 'Description line 1 Description line 2', 'test'); INSERT INTO nuts_regions (nuts3, nuts3_name, nuts2, nuts2_name, nuts1, nuts1_name) diff --git a/scripts/normalize-raw.sql b/scripts/normalize-raw.sql index 17cad66e..74ce5bf4 100644 --- a/scripts/normalize-raw.sql +++ b/scripts/normalize-raw.sql @@ -230,7 +230,7 @@ WHERE t.lot_id IS NOT NULL AND EXISTS (SELECT 1 FROM tenders te WHERE te.id = 't:' || t.unp); -- 4) Bidders — winning contractors, with a robust identity key: --- * valid ЕИК (digits-only, length 9/13) → key by ЕИК ('eik:') +-- * checksum-valid Bulgarian ЕИК → key by ЕИК ('eik:') -- * otherwise (withheld 'не се публикува', foreign id, multi-ЕИК consortium, missing) → -- key by NORMALISED NAME ('name:') -- Keying invalid ЕИК by name stops the collapse where ~595 distinct withheld-ЕИК contractors @@ -251,9 +251,9 @@ FROM ( SELECT contractor_name, eik_clean, - CASE WHEN eik_clean NOT GLOB '*[^0-9]*' AND LENGTH(eik_clean) IN (9, 13) THEN 1 ELSE 0 END AS eik_valid, + eik_valid, CASE - WHEN eik_clean NOT GLOB '*[^0-9]*' AND LENGTH(eik_clean) IN (9, 13) THEN 'eik:' || eik_clean + WHEN eik_valid = 1 THEN 'eik:' || eik_clean WHEN contractor_name IS NOT NULL AND TRIM(contractor_name) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(contractor_name, ' ', ' '), ' ', ' '))) ELSE NULL END AS bidder_key, @@ -267,8 +267,42 @@ FROM ( FROM ( SELECT contractor_name, - TRIM(CASE WHEN contractor_eik LIKE 'ЕИК %' THEN SUBSTR(contractor_eik, 5) ELSE contractor_eik END) AS eik_clean - FROM raw_contracts WHERE source LIKE 'eop:%' OR source LIKE 'ocds:%' + eik_clean, + -- Bulgarian ЕИК/Булстат control-digit (checksum) validation. A merely SYNTACTIC 9/13-digit + -- check let the fake/service code „000000001" and wrong-digit typo twins pass, collapsing + -- unrelated foreign suppliers (Elsevier + Clarivate/Web of Science + a gas consultancy + a + -- 102 EUR construction line) onto one node (#195). Enforce the real checksum so invalid codes + -- get eik_valid = 0 and fall back to the name-based key, which splits them apart again. + -- 9-digit: weight positions 1..8 by 1..8; control = sum % 11. If that is 10, re-weight by + -- 3..10; a second 10 → 0. The 9th digit must equal the control. + -- 13-digit: the leading 9 digits must themselves be a valid 9-digit ЕИК, then weight + -- positions 9..12 by 2,7,3,5 (fallback 4,9,5,7; second 10 → 0). The 13th digit + -- must equal that control. + CASE + WHEN eik_clean IS NULL OR eik_clean IN ('000000000', '0000000000000') OR eik_clean GLOB '*[^0-9]*' OR LENGTH(eik_clean) NOT IN (9, 13) THEN 0 + WHEN ( + CASE + WHEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 + WHEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 + ELSE 0 + END + ) <> CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) THEN 0 + WHEN LENGTH(eik_clean) = 9 THEN 1 + WHEN ( + CASE + WHEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 + WHEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 + ELSE 0 + END + ) = CAST(SUBSTR(eik_clean, 13, 1) AS INTEGER) THEN 1 + ELSE 0 + END AS eik_valid + FROM ( + SELECT + contractor_name, + TRIM(CASE WHEN contractor_eik LIKE 'ЕИК %' THEN SUBSTR(contractor_eik, 5) ELSE contractor_eik END) AS eik_clean + FROM raw_contracts WHERE source LIKE 'eop:%' OR source LIKE 'ocds:%' + ) ) ) WHERE bidder_key IS NOT NULL @@ -489,13 +523,40 @@ FROM ( AND c.contract_date > date(c.published_at, '+2 day') THEN 'signed_after_publication' ELSE 'ok' END AS date_flag, - CASE - WHEN TRIM(CASE WHEN c.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(c.contractor_eik, 5) ELSE c.contractor_eik END) NOT GLOB '*[^0-9]*' - AND LENGTH(TRIM(CASE WHEN c.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(c.contractor_eik, 5) ELSE c.contractor_eik END)) IN (9, 13) - THEN 'eik:' || TRIM(CASE WHEN c.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(c.contractor_eik, 5) ELSE c.contractor_eik END) - WHEN c.contractor_name IS NOT NULL AND TRIM(c.contractor_name) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(c.contractor_name, ' ', ' '), ' ', ' '))) - ELSE NULL - END AS bidder_key + ( + SELECT + CASE + WHEN eik_valid = 1 THEN 'eik:' || eik_clean + WHEN c.contractor_name IS NOT NULL AND TRIM(c.contractor_name) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(c.contractor_name, ' ', ' '), ' ', ' '))) + ELSE NULL + END + FROM ( + SELECT + eik_clean, + CASE + WHEN eik_clean IS NULL OR eik_clean IN ('000000000', '0000000000000') OR eik_clean GLOB '*[^0-9]*' OR LENGTH(eik_clean) NOT IN (9, 13) THEN 0 + WHEN ( + CASE + WHEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 + WHEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 + ELSE 0 + END + ) <> CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) THEN 0 + WHEN LENGTH(eik_clean) = 9 THEN 1 + WHEN ( + CASE + WHEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 + WHEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 + ELSE 0 + END + ) = CAST(SUBSTR(eik_clean, 13, 1) AS INTEGER) THEN 1 + ELSE 0 + END AS eik_valid + FROM ( + SELECT TRIM(CASE WHEN c.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(c.contractor_eik, 5) ELSE c.contractor_eik END) AS eik_clean + ) + ) + ) AS bidder_key FROM ( SELECT c.*, CASE @@ -757,13 +818,40 @@ SELECT 1, WHEN c.proc_est_eur > 0 AND c.eff_eur >= 10 * c.proc_est_eur THEN 'review' ELSE 'ok' END AS value_flag, - CASE - WHEN TRIM(CASE WHEN c.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(c.contractor_eik, 5) ELSE c.contractor_eik END) NOT GLOB '*[^0-9]*' - AND LENGTH(TRIM(CASE WHEN c.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(c.contractor_eik, 5) ELSE c.contractor_eik END)) IN (9, 13) - THEN 'eik:' || TRIM(CASE WHEN c.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(c.contractor_eik, 5) ELSE c.contractor_eik END) - WHEN c.contractor_name IS NOT NULL AND TRIM(c.contractor_name) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(c.contractor_name, ' ', ' '), ' ', ' '))) - ELSE NULL - END AS bidder_key + ( + SELECT + CASE + WHEN eik_valid = 1 THEN 'eik:' || eik_clean + WHEN c.contractor_name IS NOT NULL AND TRIM(c.contractor_name) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(c.contractor_name, ' ', ' '), ' ', ' '))) + ELSE NULL + END + FROM ( + SELECT + eik_clean, + CASE + WHEN eik_clean IS NULL OR eik_clean IN ('000000000', '0000000000000') OR eik_clean GLOB '*[^0-9]*' OR LENGTH(eik_clean) NOT IN (9, 13) THEN 0 + WHEN ( + CASE + WHEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 + WHEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 + ELSE 0 + END + ) <> CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) THEN 0 + WHEN LENGTH(eik_clean) = 9 THEN 1 + WHEN ( + CASE + WHEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 + WHEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 + ELSE 0 + END + ) = CAST(SUBSTR(eik_clean, 13, 1) AS INTEGER) THEN 1 + ELSE 0 + END AS eik_valid + FROM ( + SELECT TRIM(CASE WHEN c.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(c.contractor_eik, 5) ELSE c.contractor_eik END) AS eik_clean + ) + ) + ) AS bidder_key FROM ( SELECT c.*, CASE diff --git a/scripts/refresh-slice.sql b/scripts/refresh-slice.sql index e051a7ed..a145895c 100644 --- a/scripts/refresh-slice.sql +++ b/scripts/refresh-slice.sql @@ -56,10 +56,9 @@ SELECT MAX(grp), CASE WHEN MAX(grp) = 1 THEN 'consortium' ELSE 'company' END FROM ( - SELECT contractor_name, eik_clean, - CASE WHEN eik_clean NOT GLOB '*[^0-9]*' AND LENGTH(eik_clean) IN (9, 13) THEN 1 ELSE 0 END AS eik_valid, + SELECT contractor_name, eik_clean, eik_valid, CASE - WHEN eik_clean NOT GLOB '*[^0-9]*' AND LENGTH(eik_clean) IN (9, 13) THEN 'eik:' || eik_clean + WHEN eik_valid = 1 THEN 'eik:' || eik_clean WHEN contractor_name IS NOT NULL AND TRIM(contractor_name) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(contractor_name, ' ', ' '), ' ', ' '))) ELSE NULL END AS bidder_key, @@ -71,9 +70,33 @@ FROM ( THEN 1 ELSE 0 END AS grp FROM ( - SELECT contractor_name, - TRIM(CASE WHEN contractor_eik LIKE 'ЕИК %' THEN SUBSTR(contractor_eik, 5) ELSE contractor_eik END) AS eik_clean - FROM raw_contracts WHERE source LIKE 'eop:%' OR source LIKE 'ocds:%' + SELECT contractor_name, eik_clean, + -- Bulgarian ЕИК/Булстат checksum — identical to normalize-raw.sql step 4 (#195), so the + -- incremental path's bidder identity stays in parity with the full rebuild. + CASE + WHEN eik_clean IS NULL OR eik_clean IN ('000000000', '0000000000000') OR eik_clean GLOB '*[^0-9]*' OR LENGTH(eik_clean) NOT IN (9, 13) THEN 0 + WHEN ( + CASE + WHEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 + WHEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 + ELSE 0 + END + ) <> CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) THEN 0 + WHEN LENGTH(eik_clean) = 9 THEN 1 + WHEN ( + CASE + WHEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 + WHEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 + ELSE 0 + END + ) = CAST(SUBSTR(eik_clean, 13, 1) AS INTEGER) THEN 1 + ELSE 0 + END AS eik_valid + FROM ( + SELECT contractor_name, + TRIM(CASE WHEN contractor_eik LIKE 'ЕИК %' THEN SUBSTR(contractor_eik, 5) ELSE contractor_eik END) AS eik_clean + FROM raw_contracts WHERE source LIKE 'eop:%' OR source LIKE 'ocds:%' + ) ) ) WHERE bidder_key IS NOT NULL @@ -679,13 +702,40 @@ FROM ( AND c.contract_date > date(c.published_at, '+2 day') THEN 'signed_after_publication' ELSE 'ok' END AS date_flag, - CASE - WHEN TRIM(CASE WHEN c.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(c.contractor_eik, 5) ELSE c.contractor_eik END) NOT GLOB '*[^0-9]*' - AND LENGTH(TRIM(CASE WHEN c.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(c.contractor_eik, 5) ELSE c.contractor_eik END)) IN (9, 13) - THEN 'eik:' || TRIM(CASE WHEN c.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(c.contractor_eik, 5) ELSE c.contractor_eik END) - WHEN c.contractor_name IS NOT NULL AND TRIM(c.contractor_name) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(c.contractor_name, ' ', ' '), ' ', ' '))) - ELSE NULL - END AS bidder_key + ( + SELECT + CASE + WHEN eik_valid = 1 THEN 'eik:' || eik_clean + WHEN c.contractor_name IS NOT NULL AND TRIM(c.contractor_name) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(c.contractor_name, ' ', ' '), ' ', ' '))) + ELSE NULL + END + FROM ( + SELECT + eik_clean, + CASE + WHEN eik_clean IS NULL OR eik_clean IN ('000000000', '0000000000000') OR eik_clean GLOB '*[^0-9]*' OR LENGTH(eik_clean) NOT IN (9, 13) THEN 0 + WHEN ( + CASE + WHEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 + WHEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 + ELSE 0 + END + ) <> CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) THEN 0 + WHEN LENGTH(eik_clean) = 9 THEN 1 + WHEN ( + CASE + WHEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 + WHEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 + ELSE 0 + END + ) = CAST(SUBSTR(eik_clean, 13, 1) AS INTEGER) THEN 1 + ELSE 0 + END AS eik_valid + FROM ( + SELECT TRIM(CASE WHEN c.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(c.contractor_eik, 5) ELSE c.contractor_eik END) AS eik_clean + ) + ) + ) AS bidder_key FROM ( SELECT c.*, CASE @@ -927,13 +977,40 @@ FROM ( AND c.contract_date > date(c.published_at, '+2 day') THEN 'signed_after_publication' ELSE 'ok' END AS date_flag, - CASE - WHEN TRIM(CASE WHEN c.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(c.contractor_eik, 5) ELSE c.contractor_eik END) NOT GLOB '*[^0-9]*' - AND LENGTH(TRIM(CASE WHEN c.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(c.contractor_eik, 5) ELSE c.contractor_eik END)) IN (9, 13) - THEN 'eik:' || TRIM(CASE WHEN c.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(c.contractor_eik, 5) ELSE c.contractor_eik END) - WHEN c.contractor_name IS NOT NULL AND TRIM(c.contractor_name) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(c.contractor_name, ' ', ' '), ' ', ' '))) - ELSE NULL - END AS bidder_key + ( + SELECT + CASE + WHEN eik_valid = 1 THEN 'eik:' || eik_clean + WHEN c.contractor_name IS NOT NULL AND TRIM(c.contractor_name) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(c.contractor_name, ' ', ' '), ' ', ' '))) + ELSE NULL + END + FROM ( + SELECT + eik_clean, + CASE + WHEN eik_clean IS NULL OR eik_clean IN ('000000000', '0000000000000') OR eik_clean GLOB '*[^0-9]*' OR LENGTH(eik_clean) NOT IN (9, 13) THEN 0 + WHEN ( + CASE + WHEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 + WHEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 + ELSE 0 + END + ) <> CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) THEN 0 + WHEN LENGTH(eik_clean) = 9 THEN 1 + WHEN ( + CASE + WHEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 + WHEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 + ELSE 0 + END + ) = CAST(SUBSTR(eik_clean, 13, 1) AS INTEGER) THEN 1 + ELSE 0 + END AS eik_valid + FROM ( + SELECT TRIM(CASE WHEN c.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(c.contractor_eik, 5) ELSE c.contractor_eik END) AS eik_clean + ) + ) + ) AS bidder_key FROM ( SELECT c.*, CASE @@ -1244,16 +1321,43 @@ WHERE b.eik_normalized IN (SELECT eik FROM raw_ocds_parties WHERE eik IS NOT NUL OR b.id IN ( SELECT bidder_key FROM ( - SELECT contractor_name, eik_clean, + SELECT + contractor_name, CASE - WHEN eik_clean NOT GLOB '*[^0-9]*' AND LENGTH(eik_clean) IN (9, 13) THEN 'eik:' || eik_clean + WHEN eik_valid = 1 THEN 'eik:' || eik_clean WHEN contractor_name IS NOT NULL AND TRIM(contractor_name) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(contractor_name, ' ', ' '), ' ', ' '))) ELSE NULL END AS bidder_key FROM ( - SELECT contractor_name, - TRIM(CASE WHEN contractor_eik LIKE 'ЕИК %' THEN SUBSTR(contractor_eik, 5) ELSE contractor_eik END) AS eik_clean - FROM raw_contracts WHERE source LIKE 'eop:%' OR source LIKE 'ocds:%' + SELECT + contractor_name, + eik_clean, + -- Same Bulgarian ЕИК checksum as the bidder INSERT above and normalize-raw.sql (#195), + -- so the touched-bidder key matches the actual bidders.id keying. + CASE + WHEN eik_clean IS NULL OR eik_clean IN ('000000000', '0000000000000') OR eik_clean GLOB '*[^0-9]*' OR LENGTH(eik_clean) NOT IN (9, 13) THEN 0 + WHEN ( + CASE + WHEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 + WHEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 + ELSE 0 + END + ) <> CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) THEN 0 + WHEN LENGTH(eik_clean) = 9 THEN 1 + WHEN ( + CASE + WHEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 + WHEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 + ELSE 0 + END + ) = CAST(SUBSTR(eik_clean, 13, 1) AS INTEGER) THEN 1 + ELSE 0 + END AS eik_valid + FROM ( + SELECT contractor_name, + TRIM(CASE WHEN contractor_eik LIKE 'ЕИК %' THEN SUBSTR(contractor_eik, 5) ELSE contractor_eik END) AS eik_clean + FROM raw_contracts WHERE source LIKE 'eop:%' OR source LIKE 'ocds:%' + ) ) ) WHERE bidder_key IS NOT NULL From ce144331140f38a12d90a2796ff086f1e8975441 Mon Sep 17 00:00:00 2001 From: Todor Kolev Date: Fri, 17 Jul 2026 02:44:02 +0000 Subject: [PATCH 02/17] feat(etl): canonical entity fields by frequency mode, not MIN/MAX MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Authority name, bidder name, and authority type now use the most-frequent value per ЕИК with a deterministic tiebreak (mixed-case over ALL-CAPS via a Cyrillic-aware GLOB, then longer, then lexical), mirrored in normalize-raw.sql and refresh-slice.sql. Keys stay ЕИК-based so URLs are unaffected. Extracts the canonical-authority-name fix from #203 (@StanislavBG) and adds the two asymmetrically-missed siblings: bidder name (was MIN(contractor_name)) and authority type (was MAX(authority_type), which also mis-drove the state-company bucket). --- .../etl-entity-canonicalization-sql.test.ts | 183 ++++++++++++++++++ scripts/normalize-raw.sql | 145 ++++++++++++-- scripts/refresh-slice.sql | 156 +++++++++++++-- 3 files changed, 454 insertions(+), 30 deletions(-) create mode 100644 packages/db/src/etl-entity-canonicalization-sql.test.ts diff --git a/packages/db/src/etl-entity-canonicalization-sql.test.ts b/packages/db/src/etl-entity-canonicalization-sql.test.ts new file mode 100644 index 00000000..a4029351 --- /dev/null +++ b/packages/db/src/etl-entity-canonicalization-sql.test.ts @@ -0,0 +1,183 @@ +/// +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); +const schemaPath = resolve(root, 'packages/db/migrations/0000_init.sql'); +const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); +const etlPaths = [ + ['normalize-raw', resolve(root, 'scripts/normalize-raw.sql')], + ['refresh-slice', resolve(root, 'scripts/refresh-slice.sql')], +] as const; + +function sqlite(dbPath: string, sql: string): string { + return execFileSync('sqlite3', [dbPath], { input: sql, encoding: 'utf8' }); +} + +function sqliteJson(dbPath: string, sql: string): T[] { + const out = execFileSync('sqlite3', ['-json', dbPath, sql], { encoding: 'utf8' }).trim(); + return out ? (JSON.parse(out) as T[]) : []; +} + +function readScript(dbPath: string, path: string): void { + execFileSync('sqlite3', ['-bail', dbPath], { + input: `PRAGMA foreign_keys=ON;\n.read ${path}\n`, + stdio: 'pipe', + }); +} + +function withEtlDb(label: string, run: (dbPath: string) => void): void { + const dir = mkdtempSync(resolve(tmpdir(), `sigma-entity-canonicalization-${label}-`)); + const dbPath = resolve(dir, 'test.sqlite'); + try { + readScript(dbPath, schemaPath); + readScript(dbPath, stagingPath); + run(dbPath); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +describe('ETL entity canonicalization through real SQL scripts', () => { + it('uses the authority-name mode and all deterministic tiebreakers in both ETL paths', () => { + const modeEik = '300000001'; + const caseEik = '300000002'; + const lengthEik = '300000003'; + const lexicalEik = '300000004'; + + for (const [label, scriptPath] of etlPaths) { + withEtlDb(label, (dbPath) => { + sqlite( + dbPath, + `INSERT INTO raw_tenders + (source, fetched_at, unp, authority_eik, authority_name, authority_type) + VALUES + ('eop:tenders:mode-1', '2026-06-01T00:00:00Z', 'UNP-MODE-1', '${modeEik}', + 'БСУ Д-р Петър Берон', 'public'), + ('eop:tenders:mode-2', '2026-06-01T00:00:00Z', 'UNP-MODE-2', '${modeEik}', + 'МИНИСТЕРСТВО НА ОБРАЗОВАНИЕТО И НАУКАТА', 'public'), + ('eop:tenders:mode-3', '2026-06-01T00:00:00Z', 'UNP-MODE-3', '${modeEik}', + 'МИНИСТЕРСТВО НА ОБРАЗОВАНИЕТО И НАУКАТА', 'public'), + ('eop:tenders:mode-4', '2026-06-01T00:00:00Z', 'UNP-MODE-4', '${modeEik}', + 'МИНИСТЕРСТВО НА ОБРАЗОВАНИЕТО И НАУКАТА', 'public'), + ('eop:tenders:case-1', '2026-06-01T00:00:00Z', 'UNP-CASE-1', '${caseEik}', + 'TEST AUTHORITY', 'public'), + ('eop:tenders:case-2', '2026-06-01T00:00:00Z', 'UNP-CASE-2', '${caseEik}', + 'Test Authority', 'public'), + ('eop:tenders:length-1', '2026-06-01T00:00:00Z', 'UNP-LENGTH-1', '${lengthEik}', + 'Short name', 'public'), + ('eop:tenders:length-2', '2026-06-01T00:00:00Z', 'UNP-LENGTH-2', '${lengthEik}', + 'Longer authority name', 'public'), + ('eop:tenders:lexical-1', '2026-06-01T00:00:00Z', 'UNP-LEXICAL-1', '${lexicalEik}', + 'Alpha key', 'public'), + ('eop:tenders:lexical-2', '2026-06-01T00:00:00Z', 'UNP-LEXICAL-2', '${lexicalEik}', + 'Bravo key', 'public');`, + ); + + expect( + sqliteJson<{ name: string }>( + dbPath, + `SELECT MIN(authority_name) AS name FROM raw_tenders WHERE authority_eik = '${modeEik}'`, + )[0]?.name, + ).toBe('БСУ Д-р Петър Берон'); + + readScript(dbPath, scriptPath); + + expect( + sqliteJson<{ bulstat: string; name: string }>( + dbPath, + `SELECT bulstat, name FROM authorities + WHERE bulstat IN ('${modeEik}', '${caseEik}', '${lengthEik}', '${lexicalEik}') + ORDER BY bulstat`, + ), + ).toEqual([ + { bulstat: modeEik, name: 'МИНИСТЕРСТВО НА ОБРАЗОВАНИЕТО И НАУКАТА' }, + { bulstat: caseEik, name: 'Test Authority' }, + { bulstat: lengthEik, name: 'Longer authority name' }, + { bulstat: lexicalEik, name: 'Alpha key' }, + ]); + }); + } + }); + + it('uses the bidder-name mode for EIK- and name-keyed bidders in both ETL paths', () => { + const bidderEik = '400000001'; + + for (const [label, scriptPath] of etlPaths) { + withEtlDb(label, (dbPath) => { + sqlite( + dbPath, + `INSERT INTO raw_contracts + (source, fetched_at, contractor_eik, contractor_name) + VALUES + ('eop:contracts:bidder-1', '2026-06-01T00:00:00Z', '${bidderEik}', 'A Rare Bidder'), + ('eop:contracts:bidder-2', '2026-06-01T00:00:00Z', '${bidderEik}', 'Modal Bidder'), + ('eop:contracts:bidder-3', '2026-06-01T00:00:00Z', '${bidderEik}', 'Modal Bidder'), + ('eop:contracts:bidder-4', '2026-06-01T00:00:00Z', '${bidderEik}', 'Modal Bidder'), + ('eop:contracts:name-1', '2026-06-01T00:00:00Z', 'foreign-id', 'ACME'), + ('eop:contracts:name-2', '2026-06-01T00:00:00Z', 'foreign-id', 'Acme');`, + ); + + readScript(dbPath, scriptPath); + + expect( + sqliteJson<{ id: string; name: string }>( + dbPath, + `SELECT id, name FROM bidders + WHERE id IN ('eik:${bidderEik}', 'name:ACME') + ORDER BY id`, + ), + ).toEqual([ + { id: `eik:${bidderEik}`, name: 'Modal Bidder' }, + { id: 'name:ACME', name: 'Acme' }, + ]); + }); + } + }); + + it('uses the modal type for the winning authority name and buckets from raw types', () => { + const authorityEik = '500000001'; + + for (const [label, scriptPath] of etlPaths) { + withEtlDb(label, (dbPath) => { + sqlite( + dbPath, + `INSERT INTO raw_tenders + (source, fetched_at, unp, authority_eik, authority_name, authority_type) + VALUES + ('eop:tenders:type-1', '2026-06-01T00:00:00Z', 'UNP-TYPE-1', '${authorityEik}', + 'Independent authority', 'Орган на централната власт'), + ('eop:tenders:type-2', '2026-06-01T00:00:00Z', 'UNP-TYPE-2', '${authorityEik}', + 'Independent authority', 'Орган на централната власт'), + ('eop:tenders:type-3', '2026-06-01T00:00:00Z', 'UNP-TYPE-3', '${authorityEik}', + 'Independent authority', 'Орган на централната власт'), + ('eop:tenders:type-4', '2026-06-01T00:00:00Z', 'UNP-TYPE-4', '${authorityEik}', + 'Independent authority', 'Публично предприятие - сектор'), + ('eop:tenders:type-5', '2026-06-01T00:00:00Z', 'UNP-TYPE-5', '${authorityEik}', + 'A misleading label', 'Комунални услуги - вода'), + ('eop:tenders:type-6', '2026-06-01T00:00:00Z', 'UNP-TYPE-6', '${authorityEik}', + 'A misleading label', 'Комунални услуги - вода'), + ('eop:tenders:type-7', '2026-06-01T00:00:00Z', 'UNP-TYPE-7', '${authorityEik}', + 'A misleading label', 'Комунални услуги - вода');`, + ); + + readScript(dbPath, scriptPath); + + expect( + sqliteJson<{ name: string; type: string; type_group: string }>( + dbPath, + `SELECT name, type, type_group FROM authorities WHERE bulstat = '${authorityEik}'`, + )[0], + ).toEqual({ + name: 'Independent authority', + type: 'Орган на централната власт', + type_group: 'държавна компания', + }); + }); + } + }); +}); diff --git a/scripts/normalize-raw.sql b/scripts/normalize-raw.sql index 17cad66e..97ba2675 100644 --- a/scripts/normalize-raw.sql +++ b/scripts/normalize-raw.sql @@ -46,14 +46,82 @@ DELETE FROM authorities; -- 1) Authorities — dedupe on ЕИК across both contracts and tenders staging, keep a -- canonical display name and the authority type (Вид на възложителя). -INSERT OR IGNORE INTO authorities (id, name, bulstat, type) -SELECT 'auth:' || authority_eik, MIN(authority_name), authority_eik, MAX(authority_type) +-- Names use the mode per ЕИК, with deterministic presentation-quality tiebreaks. +DROP TABLE IF EXISTS authority_canonical_name; +CREATE TABLE authority_canonical_name (authority_eik TEXT PRIMARY KEY, canonical_name TEXT); +INSERT INTO authority_canonical_name (authority_eik, canonical_name) +SELECT authority_eik, authority_name +FROM ( + SELECT + authority_eik, + authority_name, + ROW_NUMBER() OVER ( + PARTITION BY authority_eik + ORDER BY + cnt DESC, + CASE WHEN authority_name GLOB '*[a-zа-я]*' THEN 0 ELSE 1 END, + LENGTH(authority_name) DESC, + authority_name + ) AS rn + FROM ( + SELECT authority_eik, authority_name, COUNT(*) AS cnt + FROM ( + SELECT authority_eik, authority_name FROM raw_contracts WHERE authority_eik IS NOT NULL + UNION ALL + SELECT authority_eik, authority_name FROM raw_tenders WHERE authority_eik IS NOT NULL + ) + WHERE authority_name IS NOT NULL AND TRIM(authority_name) <> '' + GROUP BY authority_eik, authority_name + ) +) +WHERE rn = 1; + +-- Keep the type modal within rows carrying the winning name, so the two canonical fields describe +-- the same entity when an ЕИК has been shared or reused under multiple labels. +DROP TABLE IF EXISTS authority_canonical_type; +CREATE TABLE authority_canonical_type (authority_eik TEXT PRIMARY KEY, canonical_type TEXT); +INSERT INTO authority_canonical_type (authority_eik, canonical_type) +SELECT authority_eik, authority_type FROM ( - SELECT authority_eik, authority_name, authority_type FROM raw_contracts WHERE authority_eik IS NOT NULL - UNION ALL - SELECT authority_eik, authority_name, authority_type FROM raw_tenders WHERE authority_eik IS NOT NULL + SELECT + authority_eik, + authority_type, + ROW_NUMBER() OVER ( + PARTITION BY authority_eik + ORDER BY cnt DESC, authority_type + ) AS rn + FROM ( + SELECT authority_eik, authority_type, COUNT(*) AS cnt + FROM ( + SELECT c.authority_eik, c.authority_type + FROM raw_contracts c + JOIN authority_canonical_name acn + ON acn.authority_eik = c.authority_eik AND acn.canonical_name = c.authority_name + UNION ALL + SELECT t.authority_eik, t.authority_type + FROM raw_tenders t + JOIN authority_canonical_name acn + ON acn.authority_eik = t.authority_eik AND acn.canonical_name = t.authority_name + ) + WHERE authority_type IS NOT NULL AND TRIM(authority_type) <> '' + GROUP BY authority_eik, authority_type + ) ) -GROUP BY authority_eik; +WHERE rn = 1; + +INSERT OR IGNORE INTO authorities (id, name, bulstat, type) +SELECT + 'auth:' || s.authority_eik, + COALESCE(acn.canonical_name, s.authority_eik), + s.authority_eik, + act.canonical_type +FROM ( + SELECT authority_eik FROM raw_contracts WHERE authority_eik IS NOT NULL + UNION + SELECT authority_eik FROM raw_tenders WHERE authority_eik IS NOT NULL +) s +LEFT JOIN authority_canonical_name acn ON acn.authority_eik = s.authority_eik +LEFT JOIN authority_canonical_type act ON act.authority_eik = s.authority_eik; -- 1b) Friendly authority type buckets — heuristic from name + ЗОП type (non-critical display field; -- name patterns cover Title- and UPPER-case Cyrillic since SQLite LIKE is case-sensitive for it). @@ -63,7 +131,19 @@ UPDATE authorities SET type_group = CASE WHEN name LIKE '%болница%' OR name LIKE '%БОЛНИЦА%' OR name LIKE 'МБАЛ%' OR name LIKE '%МБАЛ%' OR name LIKE '%СБАЛ%' OR name LIKE '%ДКЦ%' OR name LIKE '%лечебно заведение%' THEN 'болница' WHEN name LIKE '%университет%' OR name LIKE '%УНИВЕРСИТЕТ%' OR name LIKE '%училище%' OR name LIKE '%УЧИЛИЩЕ%' OR name LIKE '%гимназия%' OR name LIKE '%ГИМНАЗИЯ%' OR name LIKE '%детска градина%' OR name LIKE '%ДЕТСКА ГРАДИНА%' OR name LIKE '%академия%' THEN 'образование' WHEN name LIKE '%агенция%' OR name LIKE '%Агенция%' OR name LIKE '%АГЕНЦИЯ%' THEN 'агенция' - WHEN type LIKE 'Публично предприятие%' OR type LIKE 'Комунални услуги%' THEN 'държавна компания' + WHEN EXISTS ( + SELECT 1 + FROM ( + SELECT authority_eik, authority_type FROM raw_contracts + UNION ALL + SELECT authority_eik, authority_type FROM raw_tenders + ) raw_authority_types + WHERE raw_authority_types.authority_eik = authorities.bulstat + AND ( + raw_authority_types.authority_type LIKE 'Публично предприятие%' + OR raw_authority_types.authority_type LIKE 'Комунални услуги%' + ) + ) THEN 'държавна компания' ELSE 'друго' END; @@ -238,10 +318,50 @@ WHERE t.lot_id IS NOT NULL -- which keeps the bulstat UNIQUE happy). is_consortium describes the ENTITY (a JV), so it is -- name-based — a semicolon member list or ДЗЗД / ОБЕДИНЕНИЕ / КОНСОРЦИУМ in the name; the -- per-contract awarded_to_group flag lives on contracts, not here. +DROP TABLE IF EXISTS bidder_canonical_name; +CREATE TABLE bidder_canonical_name (bidder_key TEXT PRIMARY KEY, canonical_name TEXT); +INSERT INTO bidder_canonical_name (bidder_key, canonical_name) +SELECT bidder_key, contractor_name +FROM ( + SELECT + bidder_key, + contractor_name, + ROW_NUMBER() OVER ( + PARTITION BY bidder_key + ORDER BY + cnt DESC, + CASE WHEN contractor_name GLOB '*[a-zа-я]*' THEN 0 ELSE 1 END, + LENGTH(contractor_name) DESC, + contractor_name + ) AS rn + FROM ( + SELECT bidder_key, contractor_name, COUNT(*) AS cnt + FROM ( + SELECT + contractor_name, + CASE + WHEN eik_clean NOT GLOB '*[^0-9]*' AND LENGTH(eik_clean) IN (9, 13) THEN 'eik:' || eik_clean + WHEN contractor_name IS NOT NULL AND TRIM(contractor_name) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(contractor_name, ' ', ' '), ' ', ' '))) + ELSE NULL + END AS bidder_key + FROM ( + SELECT + contractor_name, + TRIM(CASE WHEN contractor_eik LIKE 'ЕИК %' THEN SUBSTR(contractor_eik, 5) ELSE contractor_eik END) AS eik_clean + FROM raw_contracts WHERE source LIKE 'eop:%' OR source LIKE 'ocds:%' + ) + ) + WHERE bidder_key IS NOT NULL + AND contractor_name IS NOT NULL AND TRIM(contractor_name) <> '' + GROUP BY bidder_key, contractor_name + ) +) +WHERE rn = 1; + INSERT OR IGNORE INTO bidders (id, name, bulstat, eik_normalized, eik_valid, is_consortium, kind) SELECT - bidder_key, - MIN(contractor_name), + b.bidder_key, + COALESCE(bcn.canonical_name, b.bidder_key), MIN(CASE WHEN eik_valid = 1 THEN eik_clean END), MIN(CASE WHEN eik_valid = 1 THEN eik_clean END), MAX(eik_valid), @@ -270,9 +390,10 @@ FROM ( TRIM(CASE WHEN contractor_eik LIKE 'ЕИК %' THEN SUBSTR(contractor_eik, 5) ELSE contractor_eik END) AS eik_clean FROM raw_contracts WHERE source LIKE 'eop:%' OR source LIKE 'ocds:%' ) -) -WHERE bidder_key IS NOT NULL -GROUP BY bidder_key; +) b +LEFT JOIN bidder_canonical_name bcn ON bcn.bidder_key = b.bidder_key +WHERE b.bidder_key IS NOT NULL +GROUP BY b.bidder_key; -- 4b) Curated public-owned winner classification. Exact EIK matches cover the allowlist; a small -- branch list handles valid 13-digit branch EIKs used by AПИ/ОПУ, ЕСО/МЕР, БНР and Информационно diff --git a/scripts/refresh-slice.sql b/scripts/refresh-slice.sql index e051a7ed..c31f9962 100644 --- a/scripts/refresh-slice.sql +++ b/scripts/refresh-slice.sql @@ -22,15 +22,85 @@ CREATE TABLE refresh_touched_authorities (authority_id TEXT PRIMARY KEY); -- @refresh-batch authorities-bidders -- ── 1) Authorities referenced by OCDS staging (new ones only; INSERT OR IGNORE) ──────────────────── -INSERT OR IGNORE INTO authorities (id, name, bulstat, type) -SELECT 'auth:' || authority_eik, MIN(authority_name), authority_eik, MAX(authority_type) +DROP TABLE IF EXISTS authority_canonical_name; +CREATE TABLE authority_canonical_name (authority_eik TEXT PRIMARY KEY, canonical_name TEXT); +INSERT INTO authority_canonical_name (authority_eik, canonical_name) +SELECT authority_eik, authority_name FROM ( - SELECT source, authority_eik, authority_name, authority_type FROM raw_contracts - UNION ALL - SELECT source, authority_eik, authority_name, authority_type FROM raw_tenders + SELECT + authority_eik, + authority_name, + ROW_NUMBER() OVER ( + PARTITION BY authority_eik + ORDER BY + cnt DESC, + CASE WHEN authority_name GLOB '*[a-zа-я]*' THEN 0 ELSE 1 END, + LENGTH(authority_name) DESC, + authority_name + ) AS rn + FROM ( + SELECT authority_eik, authority_name, COUNT(*) AS cnt + FROM ( + SELECT source, authority_eik, authority_name FROM raw_contracts + UNION ALL + SELECT source, authority_eik, authority_name FROM raw_tenders + ) + WHERE (source LIKE 'eop:%' OR source LIKE 'ocds:%') + AND authority_eik IS NOT NULL + AND authority_name IS NOT NULL AND TRIM(authority_name) <> '' + GROUP BY authority_eik, authority_name + ) ) -WHERE (source LIKE 'eop:%' OR source LIKE 'ocds:%') AND authority_eik IS NOT NULL -GROUP BY authority_eik; +WHERE rn = 1; + +DROP TABLE IF EXISTS authority_canonical_type; +CREATE TABLE authority_canonical_type (authority_eik TEXT PRIMARY KEY, canonical_type TEXT); +INSERT INTO authority_canonical_type (authority_eik, canonical_type) +SELECT authority_eik, authority_type +FROM ( + SELECT + authority_eik, + authority_type, + ROW_NUMBER() OVER ( + PARTITION BY authority_eik + ORDER BY cnt DESC, authority_type + ) AS rn + FROM ( + SELECT authority_eik, authority_type, COUNT(*) AS cnt + FROM ( + SELECT c.authority_eik, c.authority_type + FROM raw_contracts c + JOIN authority_canonical_name acn + ON acn.authority_eik = c.authority_eik AND acn.canonical_name = c.authority_name + WHERE c.source LIKE 'eop:%' OR c.source LIKE 'ocds:%' + UNION ALL + SELECT t.authority_eik, t.authority_type + FROM raw_tenders t + JOIN authority_canonical_name acn + ON acn.authority_eik = t.authority_eik AND acn.canonical_name = t.authority_name + WHERE t.source LIKE 'eop:%' OR t.source LIKE 'ocds:%' + ) + WHERE authority_type IS NOT NULL AND TRIM(authority_type) <> '' + GROUP BY authority_eik, authority_type + ) +) +WHERE rn = 1; + +INSERT OR IGNORE INTO authorities (id, name, bulstat, type) +SELECT + 'auth:' || s.authority_eik, + COALESCE(acn.canonical_name, s.authority_eik), + s.authority_eik, + act.canonical_type +FROM ( + SELECT authority_eik FROM raw_contracts + WHERE (source LIKE 'eop:%' OR source LIKE 'ocds:%') AND authority_eik IS NOT NULL + UNION + SELECT authority_eik FROM raw_tenders + WHERE (source LIKE 'eop:%' OR source LIKE 'ocds:%') AND authority_eik IS NOT NULL +) s +LEFT JOIN authority_canonical_name acn ON acn.authority_eik = s.authority_eik +LEFT JOIN authority_canonical_type act ON act.authority_eik = s.authority_eik; -- type_group for any authority still missing it (covers the rows just inserted) — same heuristic as -- normalize-raw.sql step 1b. @@ -40,16 +110,69 @@ UPDATE authorities SET type_group = CASE WHEN name LIKE '%болница%' OR name LIKE '%БОЛНИЦА%' OR name LIKE 'МБАЛ%' OR name LIKE '%МБАЛ%' OR name LIKE '%СБАЛ%' OR name LIKE '%ДКЦ%' OR name LIKE '%лечебно заведение%' THEN 'болница' WHEN name LIKE '%университет%' OR name LIKE '%УНИВЕРСИТЕТ%' OR name LIKE '%училище%' OR name LIKE '%УЧИЛИЩЕ%' OR name LIKE '%гимназия%' OR name LIKE '%ГИМНАЗИЯ%' OR name LIKE '%детска градина%' OR name LIKE '%ДЕТСКА ГРАДИНА%' OR name LIKE '%академия%' THEN 'образование' WHEN name LIKE '%агенция%' OR name LIKE '%Агенция%' OR name LIKE '%АГЕНЦИЯ%' THEN 'агенция' - WHEN type LIKE 'Публично предприятие%' OR type LIKE 'Комунални услуги%' THEN 'държавна компания' + WHEN EXISTS ( + SELECT 1 + FROM ( + SELECT source, authority_eik, authority_type FROM raw_contracts + UNION ALL + SELECT source, authority_eik, authority_type FROM raw_tenders + ) raw_authority_types + WHERE (raw_authority_types.source LIKE 'eop:%' OR raw_authority_types.source LIKE 'ocds:%') + AND raw_authority_types.authority_eik = authorities.bulstat + AND ( + raw_authority_types.authority_type LIKE 'Публично предприятие%' + OR raw_authority_types.authority_type LIKE 'Комунални услуги%' + ) + ) THEN 'държавна компания' ELSE 'друго' END WHERE type_group IS NULL; -- ── 2) Bidders referenced by OCDS staging (new ones only) — same identity rule as normalize step 4 ── +DROP TABLE IF EXISTS bidder_canonical_name; +CREATE TABLE bidder_canonical_name (bidder_key TEXT PRIMARY KEY, canonical_name TEXT); +INSERT INTO bidder_canonical_name (bidder_key, canonical_name) +SELECT bidder_key, contractor_name +FROM ( + SELECT + bidder_key, + contractor_name, + ROW_NUMBER() OVER ( + PARTITION BY bidder_key + ORDER BY + cnt DESC, + CASE WHEN contractor_name GLOB '*[a-zа-я]*' THEN 0 ELSE 1 END, + LENGTH(contractor_name) DESC, + contractor_name + ) AS rn + FROM ( + SELECT bidder_key, contractor_name, COUNT(*) AS cnt + FROM ( + SELECT + contractor_name, + CASE + WHEN eik_clean NOT GLOB '*[^0-9]*' AND LENGTH(eik_clean) IN (9, 13) THEN 'eik:' || eik_clean + WHEN contractor_name IS NOT NULL AND TRIM(contractor_name) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(contractor_name, ' ', ' '), ' ', ' '))) + ELSE NULL + END AS bidder_key + FROM ( + SELECT + contractor_name, + TRIM(CASE WHEN contractor_eik LIKE 'ЕИК %' THEN SUBSTR(contractor_eik, 5) ELSE contractor_eik END) AS eik_clean + FROM raw_contracts WHERE source LIKE 'eop:%' OR source LIKE 'ocds:%' + ) + ) + WHERE bidder_key IS NOT NULL + AND contractor_name IS NOT NULL AND TRIM(contractor_name) <> '' + GROUP BY bidder_key, contractor_name + ) +) +WHERE rn = 1; + INSERT OR IGNORE INTO bidders (id, name, bulstat, eik_normalized, eik_valid, is_consortium, kind) SELECT - bidder_key, - MIN(contractor_name), + b.bidder_key, + COALESCE(bcn.canonical_name, b.bidder_key), MIN(CASE WHEN eik_valid = 1 THEN eik_clean END), MIN(CASE WHEN eik_valid = 1 THEN eik_clean END), MAX(eik_valid), @@ -75,15 +198,12 @@ FROM ( TRIM(CASE WHEN contractor_eik LIKE 'ЕИК %' THEN SUBSTR(contractor_eik, 5) ELSE contractor_eik END) AS eik_clean FROM raw_contracts WHERE source LIKE 'eop:%' OR source LIKE 'ocds:%' ) -) -WHERE bidder_key IS NOT NULL -GROUP BY bidder_key +) b +LEFT JOIN bidder_canonical_name bcn ON bcn.bidder_key = b.bidder_key +WHERE b.bidder_key IS NOT NULL +GROUP BY b.bidder_key ON CONFLICT(id) DO UPDATE SET - name = CASE - WHEN excluded.name IS NULL THEN bidders.name - WHEN bidders.name IS NULL THEN excluded.name - ELSE min(bidders.name, excluded.name) - END, + name = excluded.name, bulstat = COALESCE(bidders.bulstat, excluded.bulstat), eik_normalized = COALESCE(bidders.eik_normalized, excluded.eik_normalized), eik_valid = max(bidders.eik_valid, excluded.eik_valid), From 5fa16cc60f867df9dc8c8122facc77ec229015c9 Mon Sep 17 00:00:00 2001 From: Todor Kolev Date: Fri, 17 Jul 2026 02:45:42 +0000 Subject: [PATCH 03/17] feat(db): attribute joint procurements to each co-authority MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Contracts whose authority_eik is a '; '-joined list of co-authorities are no longer excluded/dropped (was 579 contracts / 302M EUR vanishing). A lead authority is chosen per joint tender (co-author with the modal canonical name, fallback first ЕИК) so the contract attaches to a real authority; all co-authorities are recorded in a new contract_co_authorities bridge (migration 0002). Full contract value is attributed to the lead only in authority_totals.spent_eur (no split, no double-count); co-participation lives in a separate authority_joint_participation rollup that never enters spent_eur. Builds on the #196 diagnosis in #203 (@StanislavBG); related to #250 (@atanasster). --- packages/db/migrations/0000_init.sql | 18 ++ .../0002_contract_co_authorities.sql | 18 ++ packages/db/src/migrations.test.ts | 15 ++ scripts/normalize-raw.sql | 198 ++++++++++++++++- scripts/precompute.sql | 14 ++ scripts/refresh-slice.sql | 202 +++++++++++++++++- scripts/ship-domain.mjs | 1 + 7 files changed, 453 insertions(+), 13 deletions(-) create mode 100644 packages/db/migrations/0002_contract_co_authorities.sql diff --git a/packages/db/migrations/0000_init.sql b/packages/db/migrations/0000_init.sql index 0888ef93..7af2f46b 100644 --- a/packages/db/migrations/0000_init.sql +++ b/packages/db/migrations/0000_init.sql @@ -151,6 +151,17 @@ CREATE TABLE contracts ( created_at TEXT NOT NULL DEFAULT (datetime('now')) ); +-- Joint procurements retain one lead authority on tenders (the only authority that receives spend), +-- while this bridge records every participating authority. ordinal 0 is always the lead. +CREATE TABLE contract_co_authorities ( + contract_id TEXT NOT NULL REFERENCES contracts(id) ON DELETE CASCADE, + authority_id TEXT NOT NULL REFERENCES authorities(id), + ordinal INTEGER NOT NULL CHECK (ordinal >= 0), + PRIMARY KEY (contract_id, authority_id) +); +CREATE INDEX idx_contract_co_authorities_authority + ON contract_co_authorities(authority_id, contract_id); + -- Domain amendment history used to roll current_value/annex_count without staging. Natural-keyed so -- re-imports are idempotent; built by scripts/promote-amendments.sql from the transient amendment feed. CREATE TABLE amendments ( @@ -244,6 +255,13 @@ CREATE TABLE authority_totals ( last_date TEXT ); +-- Non-monetary joint-procurement participation rollup. It is deliberately separate from +-- authority_totals so neither spent_eur nor the ordinary lead-contract count can double-count. +CREATE TABLE authority_joint_participation ( + authority_id TEXT PRIMARY KEY REFERENCES authorities(id), + joint_contract_participations INTEGER NOT NULL +); + -- Per CPV division. Sector facet + filter counts on the list pages. CREATE TABLE sector_totals ( division TEXT PRIMARY KEY, -- 2-digit CPV division (joins @sigma/config CPV_SECTORS) diff --git a/packages/db/migrations/0002_contract_co_authorities.sql b/packages/db/migrations/0002_contract_co_authorities.sql new file mode 100644 index 00000000..a461e4e9 --- /dev/null +++ b/packages/db/migrations/0002_contract_co_authorities.sql @@ -0,0 +1,18 @@ +-- Joint procurements retain one lead authority on tenders, while this bridge records every +-- participating authority. ordinal = 0 is always the lead; remaining members keep source order. +CREATE TABLE IF NOT EXISTS contract_co_authorities ( + contract_id TEXT NOT NULL REFERENCES contracts(id) ON DELETE CASCADE, + authority_id TEXT NOT NULL REFERENCES authorities(id), + ordinal INTEGER NOT NULL CHECK (ordinal >= 0), + PRIMARY KEY (contract_id, authority_id) +); + +CREATE INDEX IF NOT EXISTS idx_contract_co_authorities_authority + ON contract_co_authorities(authority_id, contract_id); + +-- Non-monetary participation stays outside authority_totals, whose spend/count reconciliation is +-- intentionally unchanged and lead-only. +CREATE TABLE IF NOT EXISTS authority_joint_participation ( + authority_id TEXT PRIMARY KEY REFERENCES authorities(id), + joint_contract_participations INTEGER NOT NULL +); diff --git a/packages/db/src/migrations.test.ts b/packages/db/src/migrations.test.ts index 3e26faba..3815eadd 100644 --- a/packages/db/src/migrations.test.ts +++ b/packages/db/src/migrations.test.ts @@ -9,6 +9,7 @@ import { describe, expect, it } from 'vitest'; const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); const migration0 = resolve(root, 'packages/db/migrations/0000_init.sql'); const migration1 = resolve(root, 'packages/db/migrations/0001_flow_pairs_bidder_index.sql'); +const migration2 = resolve(root, 'packages/db/migrations/0002_contract_co_authorities.sql'); function sqlite(dbPath: string, sql: string): string { return execFileSync('sqlite3', [dbPath], { input: sql, encoding: 'utf8' }); @@ -27,6 +28,7 @@ describe('served migrations', () => { try { readScript(dbPath, migration0); readScript(dbPath, migration1); + readScript(dbPath, migration2); expect( sqlite( @@ -34,6 +36,19 @@ describe('served migrations', () => { "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='amendments';", ).trim(), ).toBe('1'); + + expect( + sqlite( + dbPath, + "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='contract_co_authorities';", + ).trim(), + ).toBe('1'); + expect( + sqlite( + dbPath, + "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='authority_joint_participation';", + ).trim(), + ).toBe('1'); expect( sqlite( dbPath, diff --git a/scripts/normalize-raw.sql b/scripts/normalize-raw.sql index 17cad66e..87ad12bf 100644 --- a/scripts/normalize-raw.sql +++ b/scripts/normalize-raw.sql @@ -31,13 +31,18 @@ -- every contract has a parent. bids stays empty (the data has a bid COUNT, not bids). -- Full clear in child→parent order (D1 enforces FKs). +DROP TABLE IF EXISTS joint_tender_leads; +DROP TABLE IF EXISTS joint_authority_members; +DROP TABLE IF EXISTS joint_tender_sources; DELETE FROM search_index; DELETE FROM flow_pairs; DELETE FROM company_totals; +DELETE FROM authority_joint_participation; DELETE FROM authority_totals; DELETE FROM sector_totals; DELETE FROM facet_counts; DELETE FROM home_totals; +DELETE FROM contract_co_authorities; DELETE FROM contracts; DELETE FROM lots; DELETE FROM tenders; @@ -49,12 +54,161 @@ DELETE FROM authorities; INSERT OR IGNORE INTO authorities (id, name, bulstat, type) SELECT 'auth:' || authority_eik, MIN(authority_name), authority_eik, MAX(authority_type) FROM ( - SELECT authority_eik, authority_name, authority_type FROM raw_contracts WHERE authority_eik IS NOT NULL + SELECT authority_eik, authority_name, authority_type FROM raw_contracts + WHERE authority_eik IS NOT NULL AND authority_eik NOT LIKE '%;%' UNION ALL - SELECT authority_eik, authority_name, authority_type FROM raw_tenders WHERE authority_eik IS NOT NULL + SELECT authority_eik, authority_name, authority_type FROM raw_tenders + WHERE authority_eik IS NOT NULL AND authority_eik NOT LIKE '%;%' ) GROUP BY authority_eik; +-- 1a) Joint procurements — split the parallel EIK/name lists once and reuse the mapping for tender +-- ownership and the contract bridge. The modal raw row per UNP wins when cumulative source buckets +-- repeat a tender; ties are deterministic. Existing standalone authority names stay untouched. +CREATE TABLE joint_tender_sources ( + unp TEXT PRIMARY KEY, + authority_eiks TEXT NOT NULL, + authority_name TEXT, + authority_type TEXT +); +WITH observations AS ( + SELECT unp, authority_eik, authority_name, authority_type + FROM raw_tenders + WHERE unp IS NOT NULL AND authority_eik LIKE '%;%' + UNION ALL + SELECT unp, authority_eik, authority_name, authority_type + FROM raw_contracts + WHERE unp IS NOT NULL AND authority_eik LIKE '%;%' +), ranked AS ( + SELECT unp, authority_eik, authority_name, authority_type, + ROW_NUMBER() OVER ( + PARTITION BY unp + ORDER BY COUNT(*) DESC, + CASE WHEN authority_name GLOB '*[a-zа-я]*' THEN 0 ELSE 1 END, + LENGTH(COALESCE(authority_name, '')) DESC, + authority_eik, COALESCE(authority_name, ''), COALESCE(authority_type, '') + ) AS rn + FROM observations + GROUP BY unp, authority_eik, authority_name, authority_type +) +INSERT INTO joint_tender_sources (unp, authority_eiks, authority_name, authority_type) +SELECT unp, authority_eik, authority_name, authority_type +FROM ranked +WHERE rn = 1; + +CREATE TABLE joint_authority_members ( + unp TEXT NOT NULL, + authority_id TEXT NOT NULL, + member_name TEXT, + authority_type TEXT, + source_ordinal INTEGER NOT NULL, + PRIMARY KEY (unp, authority_id) +); +WITH RECURSIVE split ( + unp, authority_eik, member_name, authority_type, source_ordinal, eik_rest, name_rest +) AS ( + SELECT + unp, + TRIM(CASE WHEN INSTR(authority_eiks, ';') > 0 + THEN SUBSTR(authority_eiks, 1, INSTR(authority_eiks, ';') - 1) ELSE authority_eiks END), + TRIM(CASE WHEN INSTR(COALESCE(authority_name, ''), ';') > 0 + THEN SUBSTR(authority_name, 1, INSTR(authority_name, ';') - 1) ELSE authority_name END), + authority_type, + 0, + CASE WHEN INSTR(authority_eiks, ';') > 0 + THEN SUBSTR(authority_eiks, INSTR(authority_eiks, ';') + 1) ELSE '' END, + CASE WHEN INSTR(COALESCE(authority_name, ''), ';') > 0 + THEN SUBSTR(authority_name, INSTR(authority_name, ';') + 1) ELSE '' END + FROM joint_tender_sources + UNION ALL + SELECT + unp, + TRIM(CASE WHEN INSTR(eik_rest, ';') > 0 + THEN SUBSTR(eik_rest, 1, INSTR(eik_rest, ';') - 1) ELSE eik_rest END), + TRIM(CASE WHEN INSTR(name_rest, ';') > 0 + THEN SUBSTR(name_rest, 1, INSTR(name_rest, ';') - 1) ELSE name_rest END), + authority_type, + source_ordinal + 1, + CASE WHEN INSTR(eik_rest, ';') > 0 + THEN SUBSTR(eik_rest, INSTR(eik_rest, ';') + 1) ELSE '' END, + CASE WHEN INSTR(name_rest, ';') > 0 + THEN SUBSTR(name_rest, INSTR(name_rest, ';') + 1) ELSE '' END + FROM split + WHERE TRIM(eik_rest) <> '' +) +INSERT OR IGNORE INTO joint_authority_members + (unp, authority_id, member_name, authority_type, source_ordinal) +SELECT unp, 'auth:' || authority_eik, NULLIF(member_name, ''), authority_type, source_ordinal +FROM split +WHERE authority_eik <> ''; + +-- A minority of co-authorities never occur standalone. Mint those real EIK identities from the +-- positionally corresponding name component; INSERT OR IGNORE preserves every standalone row. +WITH name_counts AS ( + SELECT authority_id, member_name, COUNT(*) AS uses, + ROW_NUMBER() OVER ( + PARTITION BY authority_id + ORDER BY COUNT(*) DESC, + CASE WHEN member_name GLOB '*[a-zа-я]*' THEN 0 ELSE 1 END, + LENGTH(member_name) DESC, member_name + ) AS rn + FROM joint_authority_members + WHERE member_name IS NOT NULL + GROUP BY authority_id, member_name +), member_defaults AS ( + SELECT m.authority_id, + COALESCE(n.member_name, SUBSTR(m.authority_id, 6)) AS authority_name, + MIN(SUBSTR(m.authority_id, 6)) AS authority_eik, + MAX(m.authority_type) AS authority_type + FROM joint_authority_members m + LEFT JOIN name_counts n ON n.authority_id = m.authority_id AND n.rn = 1 + GROUP BY m.authority_id +) +INSERT OR IGNORE INTO authorities (id, name, bulstat, type) +SELECT authority_id, authority_name, authority_eik, authority_type +FROM member_defaults; + +-- Lead default: match the tender's modal raw authority name to a co-authority's modal standalone +-- name. Joined/unmatched names fall back to the first EIK. This scratch-only mode does not alter +-- displayed canonical names (that remains the separate #194 concern). +CREATE TABLE joint_tender_leads ( + unp TEXT PRIMARY KEY, + authority_id TEXT NOT NULL REFERENCES authorities(id) +); +WITH standalone_observations AS ( + SELECT authority_eik, authority_name FROM raw_contracts + WHERE authority_eik IS NOT NULL AND authority_eik NOT LIKE '%;%' AND authority_name IS NOT NULL + UNION ALL + SELECT authority_eik, authority_name FROM raw_tenders + WHERE authority_eik IS NOT NULL AND authority_eik NOT LIKE '%;%' AND authority_name IS NOT NULL +), canonical_names AS ( + SELECT authority_eik, authority_name + FROM ( + SELECT authority_eik, authority_name, + ROW_NUMBER() OVER ( + PARTITION BY authority_eik + ORDER BY COUNT(*) DESC, + CASE WHEN authority_name GLOB '*[a-zа-я]*' THEN 0 ELSE 1 END, + LENGTH(authority_name) DESC, authority_name + ) AS rn + FROM standalone_observations + GROUP BY authority_eik, authority_name + ) + WHERE rn = 1 +), ranked AS ( + SELECT m.unp, m.authority_id, + ROW_NUMBER() OVER ( + PARTITION BY m.unp + ORDER BY CASE WHEN c.authority_name = s.authority_name THEN 0 ELSE 1 END, + m.source_ordinal, m.authority_id + ) AS rn + FROM joint_authority_members m + JOIN joint_tender_sources s ON s.unp = m.unp + LEFT JOIN canonical_names c ON c.authority_eik = SUBSTR(m.authority_id, 6) +) +INSERT INTO joint_tender_leads (unp, authority_id) +SELECT unp, authority_id FROM ranked WHERE rn = 1; + -- 1b) Friendly authority type buckets — heuristic from name + ЗОП type (non-critical display field; -- name patterns cover Title- and UPPER-case Cyrillic since SQLite LIKE is case-sensitive for it). UPDATE authorities SET type_group = CASE @@ -79,7 +233,8 @@ SELECT 't:' || t.unp, t.unp, COALESCE(t.procurement_subject, '(без предмет)'), - 'auth:' || t.authority_eik, + COALESCE((SELECT j.authority_id FROM joint_tender_leads j WHERE j.unp = t.unp), + 'auth:' || t.authority_eik), t.cpv_code, t.cpv_description, t.estimated_value, @@ -108,7 +263,13 @@ SELECT NULLIF(t.tender_id, '') -- raw EOP numeric tenderId from the header row FROM raw_tenders t WHERE t.lot_id IS NULL - AND EXISTS (SELECT 1 FROM authorities a WHERE a.id = 'auth:' || t.authority_eik); + AND EXISTS ( + SELECT 1 FROM authorities a + WHERE a.id = COALESCE( + (SELECT j.authority_id FROM joint_tender_leads j WHERE j.unp = t.unp), + 'auth:' || t.authority_eik + ) + ); -- 2b) Synthetic tenders — УНП that appear only in contracts (no tenders-export row), so -- every contract has a parent. Procedure type is unknown ('неизвестна'); subject/CPV/ @@ -117,7 +278,10 @@ WITH folded AS ( SELECT c.unp, MIN(c.procurement_subject) AS raw_title, - 'auth:' || MIN(c.authority_eik) AS authority_id, + COALESCE( + (SELECT j.authority_id FROM joint_tender_leads j WHERE j.unp = c.unp), + 'auth:' || MIN(c.authority_eik) + ) AS authority_id, MIN(c.cpv_code) AS cpv_code, MIN(c.estimated_value) AS estimated_value, MIN(c.currency) AS raw_currency, @@ -129,7 +293,6 @@ WITH folded AS ( WHERE 1 = 1 AND c.unp IS NOT NULL AND NOT EXISTS (SELECT 1 FROM raw_tenders t WHERE t.unp = c.unp) - AND EXISTS (SELECT 1 FROM authorities a WHERE a.id = 'auth:' || c.authority_eik) GROUP BY c.unp ) INSERT INTO tenders @@ -150,7 +313,7 @@ SELECT award_criteria, eop_tender_id FROM folded -WHERE true +WHERE EXISTS (SELECT 1 FROM authorities a WHERE a.id = folded.authority_id) ON CONFLICT(id) DO UPDATE SET title = CASE WHEN tenders.procedure_type = 'неизвестна' THEN CASE @@ -560,6 +723,22 @@ WHERE x.bidder_key IS NOT NULL AND EXISTS (SELECT 1 FROM tenders te WHERE te.id = 't:' || x.unp) AND EXISTS (SELECT 1 FROM bidders b WHERE b.id = x.bidder_key); +-- Every joint contract is linked to every real co-authority. Lead is forced to ordinal 0; the +-- remaining members retain their relative source-list order. This bridge never drives money sums. +INSERT OR IGNORE INTO contract_co_authorities (contract_id, authority_id, ordinal) +SELECT contract_id, authority_id, bridge_ordinal +FROM ( + SELECT c.id AS contract_id, m.authority_id, + ROW_NUMBER() OVER ( + PARTITION BY c.id + ORDER BY CASE WHEN m.authority_id = l.authority_id THEN 0 ELSE 1 END, + m.source_ordinal, m.authority_id + ) - 1 AS bridge_ordinal + FROM contracts c + JOIN joint_authority_members m ON c.tender_id = 't:' || m.unp + JOIN joint_tender_leads l ON l.unp = m.unp +); + -- Reconciliation guard: the final summary reports the contracts inserted alongside the surviving -- staging candidates, so a future NOT NULL/foreign-key mismatch is visible instead of hidden by -- INSERT OR IGNORE. @@ -818,6 +997,10 @@ SELECT 1, (SELECT COUNT(*) FROM contracts), datetime('now'); +DROP TABLE joint_tender_leads; +DROP TABLE joint_authority_members; +DROP TABLE joint_tender_sources; + -- Summary (last result set printed by `wrangler d1 execute`) SELECT (SELECT COUNT(*) FROM authorities) AS authorities, @@ -827,6 +1010,7 @@ SELECT (SELECT COUNT(*) FROM bidders WHERE eik_valid = 0) AS bidders_name_keyed, (SELECT COUNT(*) FROM bidders WHERE kind = 'consortium') AS consortia, (SELECT COUNT(*) FROM contracts) AS contracts, + (SELECT COUNT(*) FROM contract_co_authorities) AS joint_participations, (SELECT contract_candidates FROM pipeline_stats) AS contract_candidates, (SELECT COUNT(*) FROM contracts WHERE value_flag = 'value_suspect') AS value_suspect, (SELECT COUNT(*) FROM contracts WHERE value_flag = 'annex_suspect') AS annex_suspect, diff --git a/scripts/precompute.sql b/scripts/precompute.sql index d52642d1..68b4c313 100644 --- a/scripts/precompute.sql +++ b/scripts/precompute.sql @@ -77,6 +77,8 @@ CREATE TABLE IF NOT EXISTS authority_totals ( eu_eur REAL NOT NULL DEFAULT 0, first_date TEXT, last_date TEXT ); DELETE FROM authority_totals; +-- Spend and the ordinary contract count remain lead-only, preserving the reconciliation invariant: +-- SUM(authority_totals.spent_eur) = the tender-authority-attributed contract sum. INSERT INTO authority_totals (authority_id, name, type_group, settlement, region, spent_eur, contracts, suppliers, avg_eur, eu_eur, first_date, last_date) SELECT a.id, a.name, a.type_group, a.settlement, a.region, SUM(c.amount_eur), COUNT(*), COUNT(DISTINCT c.bidder_id), SUM(c.amount_eur) / COUNT(*), @@ -90,6 +92,18 @@ UPDATE authority_totals SET primary_sector = ( WHERE t.authority_id = authority_totals.authority_id AND c.amount_eur IS NOT NULL AND COALESCE(t.cpv_code,'') <> '' GROUP BY substr(t.cpv_code, 1, 2) ORDER BY SUM(c.amount_eur) DESC, substr(t.cpv_code, 1, 2) LIMIT 1); +-- Separate, non-monetary participation metric. It counts every joint-contract bridge association, +-- including the lead, and never feeds authority_totals.spent_eur or its ordinary contract count. +CREATE TABLE IF NOT EXISTS authority_joint_participation ( + authority_id TEXT PRIMARY KEY REFERENCES authorities(id), + joint_contract_participations INTEGER NOT NULL +); +DELETE FROM authority_joint_participation; +INSERT INTO authority_joint_participation (authority_id, joint_contract_participations) +SELECT authority_id, COUNT(*) +FROM contract_co_authorities +GROUP BY authority_id; + -- home_totals uses the browsable leaderboard grains for authority/bidder counts, and the same -- freshness definition as refresh-slice.sql: latest in-corpus signed contract date. INSERT INTO home_totals (id, contracts, value_eur, authorities, bidders, suspect, first_date, last_date, as_of, refreshed_at) diff --git a/scripts/refresh-slice.sql b/scripts/refresh-slice.sql index e051a7ed..ed703db0 100644 --- a/scripts/refresh-slice.sql +++ b/scripts/refresh-slice.sql @@ -16,6 +16,9 @@ CREATE INDEX IF NOT EXISTS idx_contracts_tender_id ON contracts(tender_id); DROP TABLE IF EXISTS refresh_touched_contracts; DROP TABLE IF EXISTS refresh_touched_bidders; DROP TABLE IF EXISTS refresh_touched_authorities; +DROP TABLE IF EXISTS refresh_joint_tender_leads; +DROP TABLE IF EXISTS refresh_joint_authority_members; +DROP TABLE IF EXISTS refresh_joint_tender_sources; CREATE TABLE refresh_touched_contracts (id TEXT PRIMARY KEY); CREATE TABLE refresh_touched_bidders (bidder_id TEXT PRIMARY KEY); CREATE TABLE refresh_touched_authorities (authority_id TEXT PRIMARY KEY); @@ -29,9 +32,153 @@ FROM ( UNION ALL SELECT source, authority_eik, authority_name, authority_type FROM raw_tenders ) -WHERE (source LIKE 'eop:%' OR source LIKE 'ocds:%') AND authority_eik IS NOT NULL +WHERE (source LIKE 'eop:%' OR source LIKE 'ocds:%') + AND authority_eik IS NOT NULL + AND authority_eik NOT LIKE '%;%' GROUP BY authority_eik; +-- Split joint EIK/name lists once for lead selection and bridge population. This is the scoped +-- mirror of normalize-raw.sql step 1a; it deliberately does not rewrite existing display names. +CREATE TABLE refresh_joint_tender_sources ( + unp TEXT PRIMARY KEY, + authority_eiks TEXT NOT NULL, + authority_name TEXT, + authority_type TEXT +); +WITH observations AS ( + SELECT unp, authority_eik, authority_name, authority_type + FROM raw_tenders + WHERE unp IS NOT NULL AND authority_eik LIKE '%;%' + AND (source LIKE 'eop:%' OR source LIKE 'ocds:%') + UNION ALL + SELECT unp, authority_eik, authority_name, authority_type + FROM raw_contracts + WHERE unp IS NOT NULL AND authority_eik LIKE '%;%' + AND (source LIKE 'eop:%' OR source LIKE 'ocds:%') +), ranked AS ( + SELECT unp, authority_eik, authority_name, authority_type, + ROW_NUMBER() OVER ( + PARTITION BY unp + ORDER BY COUNT(*) DESC, + CASE WHEN authority_name GLOB '*[a-zа-я]*' THEN 0 ELSE 1 END, + LENGTH(COALESCE(authority_name, '')) DESC, + authority_eik, COALESCE(authority_name, ''), COALESCE(authority_type, '') + ) AS rn + FROM observations + GROUP BY unp, authority_eik, authority_name, authority_type +) +INSERT INTO refresh_joint_tender_sources (unp, authority_eiks, authority_name, authority_type) +SELECT unp, authority_eik, authority_name, authority_type FROM ranked WHERE rn = 1; + +CREATE TABLE refresh_joint_authority_members ( + unp TEXT NOT NULL, + authority_id TEXT NOT NULL, + member_name TEXT, + authority_type TEXT, + source_ordinal INTEGER NOT NULL, + PRIMARY KEY (unp, authority_id) +); +WITH RECURSIVE split ( + unp, authority_eik, member_name, authority_type, source_ordinal, eik_rest, name_rest +) AS ( + SELECT + unp, + TRIM(CASE WHEN INSTR(authority_eiks, ';') > 0 + THEN SUBSTR(authority_eiks, 1, INSTR(authority_eiks, ';') - 1) ELSE authority_eiks END), + TRIM(CASE WHEN INSTR(COALESCE(authority_name, ''), ';') > 0 + THEN SUBSTR(authority_name, 1, INSTR(authority_name, ';') - 1) ELSE authority_name END), + authority_type, + 0, + CASE WHEN INSTR(authority_eiks, ';') > 0 + THEN SUBSTR(authority_eiks, INSTR(authority_eiks, ';') + 1) ELSE '' END, + CASE WHEN INSTR(COALESCE(authority_name, ''), ';') > 0 + THEN SUBSTR(authority_name, INSTR(authority_name, ';') + 1) ELSE '' END + FROM refresh_joint_tender_sources + UNION ALL + SELECT + unp, + TRIM(CASE WHEN INSTR(eik_rest, ';') > 0 + THEN SUBSTR(eik_rest, 1, INSTR(eik_rest, ';') - 1) ELSE eik_rest END), + TRIM(CASE WHEN INSTR(name_rest, ';') > 0 + THEN SUBSTR(name_rest, 1, INSTR(name_rest, ';') - 1) ELSE name_rest END), + authority_type, + source_ordinal + 1, + CASE WHEN INSTR(eik_rest, ';') > 0 + THEN SUBSTR(eik_rest, INSTR(eik_rest, ';') + 1) ELSE '' END, + CASE WHEN INSTR(name_rest, ';') > 0 + THEN SUBSTR(name_rest, INSTR(name_rest, ';') + 1) ELSE '' END + FROM split + WHERE TRIM(eik_rest) <> '' +) +INSERT OR IGNORE INTO refresh_joint_authority_members + (unp, authority_id, member_name, authority_type, source_ordinal) +SELECT unp, 'auth:' || authority_eik, NULLIF(member_name, ''), authority_type, source_ordinal +FROM split +WHERE authority_eik <> ''; + +WITH name_counts AS ( + SELECT authority_id, member_name, + ROW_NUMBER() OVER ( + PARTITION BY authority_id + ORDER BY COUNT(*) DESC, + CASE WHEN member_name GLOB '*[a-zа-я]*' THEN 0 ELSE 1 END, + LENGTH(member_name) DESC, member_name + ) AS rn + FROM refresh_joint_authority_members + WHERE member_name IS NOT NULL + GROUP BY authority_id, member_name +), member_defaults AS ( + SELECT m.authority_id, + COALESCE(n.member_name, SUBSTR(m.authority_id, 6)) AS authority_name, + MIN(SUBSTR(m.authority_id, 6)) AS authority_eik, + MAX(m.authority_type) AS authority_type + FROM refresh_joint_authority_members m + LEFT JOIN name_counts n ON n.authority_id = m.authority_id AND n.rn = 1 + GROUP BY m.authority_id +) +INSERT OR IGNORE INTO authorities (id, name, bulstat, type) +SELECT authority_id, authority_name, authority_eik, authority_type FROM member_defaults; + +CREATE TABLE refresh_joint_tender_leads ( + unp TEXT PRIMARY KEY, + authority_id TEXT NOT NULL REFERENCES authorities(id) +); +WITH standalone_observations AS ( + SELECT authority_eik, authority_name FROM raw_contracts + WHERE authority_eik IS NOT NULL AND authority_eik NOT LIKE '%;%' AND authority_name IS NOT NULL + AND (source LIKE 'eop:%' OR source LIKE 'ocds:%') + UNION ALL + SELECT authority_eik, authority_name FROM raw_tenders + WHERE authority_eik IS NOT NULL AND authority_eik NOT LIKE '%;%' AND authority_name IS NOT NULL + AND (source LIKE 'eop:%' OR source LIKE 'ocds:%') +), canonical_names AS ( + SELECT authority_eik, authority_name + FROM ( + SELECT authority_eik, authority_name, + ROW_NUMBER() OVER ( + PARTITION BY authority_eik + ORDER BY COUNT(*) DESC, + CASE WHEN authority_name GLOB '*[a-zа-я]*' THEN 0 ELSE 1 END, + LENGTH(authority_name) DESC, authority_name + ) AS rn + FROM standalone_observations + GROUP BY authority_eik, authority_name + ) + WHERE rn = 1 +), ranked AS ( + SELECT m.unp, m.authority_id, + ROW_NUMBER() OVER ( + PARTITION BY m.unp + ORDER BY CASE WHEN c.authority_name = s.authority_name THEN 0 ELSE 1 END, + m.source_ordinal, m.authority_id + ) AS rn + FROM refresh_joint_authority_members m + JOIN refresh_joint_tender_sources s ON s.unp = m.unp + LEFT JOIN canonical_names c ON c.authority_eik = SUBSTR(m.authority_id, 6) +) +INSERT INTO refresh_joint_tender_leads (unp, authority_id) +SELECT unp, authority_id FROM ranked WHERE rn = 1; + -- type_group for any authority still missing it (covers the rows just inserted) — same heuristic as -- normalize-raw.sql step 1b. UPDATE authorities SET type_group = CASE @@ -147,7 +294,8 @@ SELECT 't:' || t.unp, t.unp, COALESCE(t.procurement_subject, '(без предмет)'), - 'auth:' || t.authority_eik, + COALESCE((SELECT j.authority_id FROM refresh_joint_tender_leads j WHERE j.unp = t.unp), + 'auth:' || t.authority_eik), t.cpv_code, t.cpv_description, t.estimated_value, @@ -176,7 +324,13 @@ SELECT NULLIF(t.tender_id, '') -- raw EOP numeric tenderId from the header row FROM raw_tenders t WHERE t.lot_id IS NULL - AND EXISTS (SELECT 1 FROM authorities a WHERE a.id = 'auth:' || t.authority_eik) + AND EXISTS ( + SELECT 1 FROM authorities a + WHERE a.id = COALESCE( + (SELECT j.authority_id FROM refresh_joint_tender_leads j WHERE j.unp = t.unp), + 'auth:' || t.authority_eik + ) + ) ON CONFLICT(id) DO UPDATE SET source_id = CASE WHEN tenders.procedure_type = 'неизвестна' THEN excluded.source_id ELSE tenders.source_id END, title = CASE WHEN tenders.procedure_type = 'неизвестна' THEN excluded.title ELSE tenders.title END, @@ -353,7 +507,10 @@ WITH folded AS ( SELECT c.unp, MIN(c.procurement_subject) AS raw_title, - 'auth:' || MIN(c.authority_eik) AS authority_id, + COALESCE( + (SELECT j.authority_id FROM refresh_joint_tender_leads j WHERE j.unp = c.unp), + 'auth:' || MIN(c.authority_eik) + ) AS authority_id, MIN(c.cpv_code) AS cpv_code, MIN(c.estimated_value) AS estimated_value, MIN(c.currency) AS raw_currency, @@ -365,7 +522,6 @@ WITH folded AS ( WHERE (c.source LIKE 'eop:%' OR c.source LIKE 'ocds:%') AND c.unp IS NOT NULL AND NOT EXISTS (SELECT 1 FROM raw_tenders t WHERE t.unp = c.unp) - AND EXISTS (SELECT 1 FROM authorities a WHERE a.id = 'auth:' || c.authority_eik) GROUP BY c.unp ) INSERT INTO tenders @@ -386,7 +542,7 @@ SELECT award_criteria, eop_tender_id FROM folded -WHERE true +WHERE EXISTS (SELECT 1 FROM authorities a WHERE a.id = folded.authority_id) ON CONFLICT(id) DO UPDATE SET title = CASE WHEN tenders.procedure_type = 'неизвестна' THEN CASE @@ -491,6 +647,10 @@ SELECT DISTINCT t.authority_id FROM contracts c JOIN tenders t ON t.id = c.tender_id WHERE c.id IN (SELECT id FROM refresh_touched_contracts) AND t.authority_id IS NOT NULL; +INSERT OR IGNORE INTO refresh_touched_authorities (authority_id) +SELECT DISTINCT ca.authority_id +FROM contract_co_authorities ca +WHERE ca.contract_id IN (SELECT id FROM refresh_touched_contracts); DELETE FROM contracts WHERE id IN ( @@ -997,6 +1157,24 @@ WHERE x.bidder_key IS NOT NULL AND c2.bidder_id = x.bidder_key ); +-- Rebuild joint-participation links for contracts represented by this window. Contract deletes +-- cascade old bridge rows; this inserts every member again with the lead forced to ordinal 0. +INSERT OR IGNORE INTO contract_co_authorities (contract_id, authority_id, ordinal) +SELECT contract_id, authority_id, bridge_ordinal +FROM ( + SELECT c.id AS contract_id, m.authority_id, + ROW_NUMBER() OVER ( + PARTITION BY c.id + ORDER BY CASE WHEN m.authority_id = l.authority_id THEN 0 ELSE 1 END, + m.source_ordinal, m.authority_id + ) - 1 AS bridge_ordinal + FROM contracts c + JOIN refresh_joint_authority_members m ON c.tender_id = 't:' || m.unp + JOIN refresh_joint_tender_leads l ON l.unp = m.unp +); +INSERT OR IGNORE INTO refresh_touched_authorities (authority_id) +SELECT DISTINCT authority_id FROM refresh_joint_authority_members; + UPDATE tenders SET status = 'awarded' WHERE status <> 'awarded' @@ -1291,6 +1469,15 @@ UPDATE authority_totals SET primary_sector = ( GROUP BY substr(t.cpv_code, 1, 2) ORDER BY SUM(c.amount_eur) DESC, substr(t.cpv_code, 1, 2) LIMIT 1) WHERE authority_id IN (SELECT authority_id FROM refresh_touched_authorities); +-- Joint participation is a separate non-monetary rollup and never enters lead spend/count totals. +DELETE FROM authority_joint_participation +WHERE authority_id IN (SELECT authority_id FROM refresh_touched_authorities); +INSERT INTO authority_joint_participation (authority_id, joint_contract_participations) +SELECT authority_id, COUNT(*) +FROM contract_co_authorities +WHERE authority_id IN (SELECT authority_id FROM refresh_touched_authorities) +GROUP BY authority_id; + -- @refresh-batch flow-pairs DELETE FROM flow_pairs; INSERT INTO flow_pairs (authority_id, bidder_id, authority_name, bidder_name, bidder_kind, won_eur, contracts) @@ -1364,6 +1551,9 @@ SELECT 'eu', CASE WHEN c.eu_funded = 1 THEN '1' ELSE '0' END, COUNT(*), COALESCE FROM contracts c GROUP BY CASE WHEN c.eu_funded = 1 THEN '1' ELSE '0' END; -- @refresh-batch cleanup +DROP TABLE IF EXISTS refresh_joint_tender_leads; +DROP TABLE IF EXISTS refresh_joint_authority_members; +DROP TABLE IF EXISTS refresh_joint_tender_sources; DROP TABLE IF EXISTS refresh_touched_contracts; DROP TABLE IF EXISTS refresh_touched_bidders; DROP TABLE IF EXISTS refresh_touched_authorities; diff --git a/scripts/ship-domain.mjs b/scripts/ship-domain.mjs index 531751fd..3d771675 100755 --- a/scripts/ship-domain.mjs +++ b/scripts/ship-domain.mjs @@ -16,6 +16,7 @@ const TABLES = [ 'tenders', 'lots', 'contracts', + 'contract_co_authorities', 'amendments', 'parties', 'fx_rates', From 45b100a8c594aa3bb72dd539100c3255601bc9b8 Mon Sep 17 00:00:00 2001 From: Todor Kolev Date: Fri, 17 Jul 2026 22:00:28 +0000 Subject: [PATCH 04/17] feat(etl,web): preserve verbatim ordering-unit name beside the canonical authority MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Under #194 the authority display name is canonicalized to the frequency mode per ЕИК, which collapses genuine sub-units that share one ЕИК (e.g. a school placed under МОН's ЕИК keeps the ministry name). Additionally keep the raw per-row "Възложител" verbatim so no ordering unit is lost. - add contracts.ordering_unit_name / tenders.ordering_unit_name, filled from the raw authority_name in normalize-raw and both refresh-slice paths - getContract surfaces it on the contract page only when it differs from the canonical name after a Cyrillic-safe fold (case + whitespace), so spelling variants stay hidden and real sub-units show - /contracts/:id.json sourceNames.authority is now genuinely verbatim; it previously served the canonical name despite the "verbatim" contract Extends #251; credit StanislavBG's #203 identity work. --- apps/web/app/routes/contract.tsx | 5 +++ packages/api-contract/src/index.ts | 1 + packages/db/migrations/0000_init.sql | 2 ++ .../etl-entity-canonicalization-sql.test.ts | 33 +++++++++++++++++-- packages/db/src/queries/details.ts | 17 +++++++++- scripts/normalize-raw.sql | 7 ++-- scripts/refresh-slice.sql | 12 +++++-- 7 files changed, 69 insertions(+), 8 deletions(-) diff --git a/apps/web/app/routes/contract.tsx b/apps/web/app/routes/contract.tsx index 256617de..ce7855af 100644 --- a/apps/web/app/routes/contract.tsx +++ b/apps/web/app/routes/contract.tsx @@ -269,6 +269,11 @@ export default function Contract({ loaderData }: Route.ComponentProps) {

{c.authority.name}

+ {c.authority.orderingUnit && ( +

+ Възложител по документа: {c.authority.orderingUnit} +

+ )}

{c.authority.typeLabel && {c.authority.typeLabel}} {c.authority.settlement && <> {c.authority.settlement}} diff --git a/packages/api-contract/src/index.ts b/packages/api-contract/src/index.ts index ad7d6d1d..4d840867 100644 --- a/packages/api-contract/src/index.ts +++ b/packages/api-contract/src/index.ts @@ -235,6 +235,7 @@ export interface ContractParty { slug: string; name: string; displayName: string; + orderingUnit: string | null; kind?: EntityKind; typeLabel: string | null; settlement: string | null; diff --git a/packages/db/migrations/0000_init.sql b/packages/db/migrations/0000_init.sql index 0888ef93..51127e8a 100644 --- a/packages/db/migrations/0000_init.sql +++ b/packages/db/migrations/0000_init.sql @@ -42,6 +42,7 @@ CREATE TABLE tenders ( source_id TEXT NOT NULL UNIQUE, -- УНП (ЦАИС ЕОП identifier) title TEXT NOT NULL, authority_id TEXT NOT NULL REFERENCES authorities(id), + ordering_unit_name TEXT, -- verbatim "Възложител" (raw authority_name) on this row; preserves sub-unit identity collapsed by canonical authorities.name (mode) cpv_code TEXT, cpv_description TEXT, -- human-readable CPV label (no external dictionary needed) estimated_value REAL, @@ -112,6 +113,7 @@ CREATE TABLE contracts ( id TEXT PRIMARY KEY, -- 'c:' || staging row id tender_id TEXT NOT NULL REFERENCES tenders(id), bidder_id TEXT NOT NULL REFERENCES bidders(id), + ordering_unit_name TEXT, -- verbatim "Възложител" (raw authority_name) on this row; preserves sub-unit identity collapsed by canonical authorities.name (mode) amount REAL NOT NULL, -- as-recorded headline value in `currency` (display); signing for annex_suspect currency TEXT NOT NULL DEFAULT 'BGN', signed_at TEXT, diff --git a/packages/db/src/etl-entity-canonicalization-sql.test.ts b/packages/db/src/etl-entity-canonicalization-sql.test.ts index a4029351..ca85461b 100644 --- a/packages/db/src/etl-entity-canonicalization-sql.test.ts +++ b/packages/db/src/etl-entity-canonicalization-sql.test.ts @@ -44,7 +44,7 @@ function withEtlDb(label: string, run: (dbPath: string) => void): void { describe('ETL entity canonicalization through real SQL scripts', () => { it('uses the authority-name mode and all deterministic tiebreakers in both ETL paths', () => { - const modeEik = '300000001'; + const modeEik = '000695114'; const caseEik = '300000002'; const lengthEik = '300000003'; const lexicalEik = '300000004'; @@ -75,7 +75,18 @@ describe('ETL entity canonicalization through real SQL scripts', () => { ('eop:tenders:lexical-1', '2026-06-01T00:00:00Z', 'UNP-LEXICAL-1', '${lexicalEik}', 'Alpha key', 'public'), ('eop:tenders:lexical-2', '2026-06-01T00:00:00Z', 'UNP-LEXICAL-2', '${lexicalEik}', - 'Bravo key', 'public');`, + 'Bravo key', 'public'); + + INSERT INTO raw_contracts + (source, fetched_at, unp, authority_eik, authority_name, contract_number, + contract_date, signing_value, currency, contractor_eik, contractor_name) + VALUES + ('eop:contracts:subunit', '2026-06-01T00:00:00Z', 'UNP-MODE-1', '${modeEik}', + 'СУ „Христо Ботев" гр. Пловдив', 'CONTRACT-SUBUNIT', '2026-06-01', 1000, 'BGN', + '400000011', 'Subunit Bidder'), + ('eop:contracts:case', '2026-06-01T00:00:00Z', 'UNP-MODE-2', '${modeEik}', + 'Министерство на образованието и науката', 'CONTRACT-CASE', '2026-06-01', 1000, + 'BGN', '400000012', 'Case Bidder');`, ); expect( @@ -100,6 +111,24 @@ describe('ETL entity canonicalization through real SQL scripts', () => { { bulstat: lengthEik, name: 'Longer authority name' }, { bulstat: lexicalEik, name: 'Alpha key' }, ]); + + expect( + sqliteJson<{ contract_number: string; ordering_unit_name: string }>( + dbPath, + `SELECT contract_number, ordering_unit_name FROM contracts + WHERE contract_number IN ('CONTRACT-SUBUNIT', 'CONTRACT-CASE') + ORDER BY contract_number`, + ), + ).toEqual([ + { + contract_number: 'CONTRACT-CASE', + ordering_unit_name: 'Министерство на образованието и науката', + }, + { + contract_number: 'CONTRACT-SUBUNIT', + ordering_unit_name: 'СУ „Христо Ботев" гр. Пловдив', + }, + ]); }); } }); diff --git a/packages/db/src/queries/details.ts b/packages/db/src/queries/details.ts index 8b3604f5..71f8347e 100644 --- a/packages/db/src/queries/details.ts +++ b/packages/db/src/queries/details.ts @@ -362,6 +362,8 @@ export async function getAuthority( // ── Contract ────────────────────────────────────────────────────────────────────────────────── +const foldName = (s: string): string => s.toLocaleLowerCase('bg').replace(/\s+/g, ' ').trim(); + interface ContractDetailRow { id: string; tender_id: string; @@ -408,6 +410,7 @@ interface ContractDetailRow { // authority authority_id: string; authority_name: string; + source_authority_name: string | null; authority_type_group: string | null; authority_settlement: string | null; // bidder @@ -466,6 +469,7 @@ export async function getContract( c.signing_value_eur, c.current_value_eur, c.value_flag, c.date_flag, c.bids_received, c.bids_rejected, c.bids_sme, c.bids_non_eea, c.subcontractor_eik, c.subcontractor_name, c.subcontract_value, c.currency AS contract_currency, + c.ordering_unit_name AS source_authority_name, t.title, t.source_id AS unp, t.procedure_type, t.cpv_code, t.cpv_description, t.num_lots, t.eop_tender_id, t.estimated_value, t.currency AS tender_currency, t.start_date, t.end_date, @@ -602,6 +606,10 @@ export async function getContract( slug: authoritySlug(r.authority_id), name: cleanName(r.authority_name), displayName: cleanName(r.authority_name), + orderingUnit: + r.source_authority_name && foldName(r.source_authority_name) !== foldName(r.authority_name) + ? cleanName(r.source_authority_name) + : null, typeLabel: typeLabel(r.authority_type_group), settlement: r.authority_settlement, eik: authoritySlug(r.authority_id), @@ -613,6 +621,7 @@ export async function getContract( slug: companySlug(r.bidder_id), name: cleanName(r.bidder_name), displayName: entityName(cleanName(r.bidder_name), r.bidder_kind), + orderingUnit: null, kind: r.bidder_kind, typeLabel: null, settlement: r.bidder_settlement, @@ -696,5 +705,11 @@ export async function getContract( amendments, }; - return { ...detail, sourceNames: { authority: r.authority_name, bidder: r.bidder_name } }; + return { + ...detail, + sourceNames: { + authority: r.source_authority_name ?? r.authority_name, + bidder: r.bidder_name, + }, + }; } diff --git a/scripts/normalize-raw.sql b/scripts/normalize-raw.sql index 97ba2675..0b4f1018 100644 --- a/scripts/normalize-raw.sql +++ b/scripts/normalize-raw.sql @@ -150,7 +150,7 @@ END; -- 2a) Tenders — the header row of each procurement (lot_id IS NULL): one per УНП, carrying -- procedure type, CPV, the procurement-level estimated value, lot count and authority. INSERT OR IGNORE INTO tenders - (id, source_id, title, authority_id, cpv_code, cpv_description, estimated_value, currency, + (id, source_id, title, authority_id, ordering_unit_name, cpv_code, cpv_description, estimated_value, currency, procedure_type, contract_kind, num_lots, status, published_at, deadline_at, legal_basis, award_criteria, main_activity, notice_type, place_of_performance, start_date, end_date, duration, duration_unit, @@ -160,6 +160,7 @@ SELECT t.unp, COALESCE(t.procurement_subject, '(без предмет)'), 'auth:' || t.authority_eik, + NULLIF(TRIM(t.authority_name), ''), t.cpv_code, t.cpv_description, t.estimated_value, @@ -443,7 +444,7 @@ SET ownership_kind = ( -- fx_rate carries the applied rate on the row (amount * fx_rate = amount_eur), so the original value, -- the rate, and the EUR value are all auditable without joining fx_rates. INSERT OR IGNORE INTO contracts - (id, tender_id, bidder_id, amount, currency, signed_at, + (id, tender_id, bidder_id, ordering_unit_name, amount, currency, signed_at, contract_number, signing_value, current_value, annex_count, eu_funded, bids_received, contract_kind, awarded_to_group, value_flag, date_flag, amount_eur, fx_converted, fx_rate, lot_id, document_number, published_at, contract_subject, @@ -461,6 +462,7 @@ SELECT END, 't:' || x.unp, x.bidder_key, + NULLIF(TRIM(x.authority_name_raw), ''), x.display_native, COALESCE(x.currency, 'BGN'), x.contract_date, @@ -619,6 +621,7 @@ FROM ( END AS bidder_key FROM ( SELECT c.*, + c.authority_name AS authority_name_raw, CASE WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN COALESCE(c.current_value, c.signing_value) WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'BGN' THEN COALESCE(c.current_value, c.signing_value) / 1.95583 diff --git a/scripts/refresh-slice.sql b/scripts/refresh-slice.sql index c31f9962..a79a2ea7 100644 --- a/scripts/refresh-slice.sql +++ b/scripts/refresh-slice.sql @@ -258,7 +258,7 @@ WHERE c.id IN (SELECT id FROM refresh_touched_contracts) -- @refresh-batch tenders -- EOP tender headers and lots loaded since the last full normalize. INSERT INTO tenders - (id, source_id, title, authority_id, cpv_code, cpv_description, estimated_value, currency, + (id, source_id, title, authority_id, ordering_unit_name, cpv_code, cpv_description, estimated_value, currency, procedure_type, contract_kind, num_lots, status, published_at, deadline_at, legal_basis, award_criteria, main_activity, notice_type, place_of_performance, start_date, end_date, duration, duration_unit, @@ -268,6 +268,7 @@ SELECT t.unp, COALESCE(t.procurement_subject, '(без предмет)'), 'auth:' || t.authority_eik, + NULLIF(TRIM(t.authority_name), ''), t.cpv_code, t.cpv_description, t.estimated_value, @@ -301,6 +302,7 @@ ON CONFLICT(id) DO UPDATE SET source_id = CASE WHEN tenders.procedure_type = 'неизвестна' THEN excluded.source_id ELSE tenders.source_id END, title = CASE WHEN tenders.procedure_type = 'неизвестна' THEN excluded.title ELSE tenders.title END, authority_id = CASE WHEN tenders.procedure_type = 'неизвестна' THEN excluded.authority_id ELSE tenders.authority_id END, + ordering_unit_name = excluded.ordering_unit_name, cpv_code = CASE WHEN tenders.procedure_type = 'неизвестна' THEN COALESCE(excluded.cpv_code, tenders.cpv_code) ELSE tenders.cpv_code END, cpv_description = CASE WHEN tenders.procedure_type = 'неизвестна' THEN COALESCE(excluded.cpv_description, tenders.cpv_description) ELSE tenders.cpv_description END, estimated_value = CASE WHEN tenders.procedure_type = 'неизвестна' THEN COALESCE(excluded.estimated_value, tenders.estimated_value) ELSE tenders.estimated_value END, @@ -635,7 +637,7 @@ WHERE id IN ( ) ); INSERT OR IGNORE INTO contracts - (id, tender_id, bidder_id, amount, currency, signed_at, contract_number, signing_value, current_value, + (id, tender_id, bidder_id, ordering_unit_name, amount, currency, signed_at, contract_number, signing_value, current_value, annex_count, eu_funded, bids_received, contract_kind, awarded_to_group, value_flag, date_flag, amount_eur, fx_converted, fx_rate, signing_value_eur, current_value_eur, lot_id, document_number, published_at, contract_subject, @@ -648,6 +650,7 @@ SELECT COALESCE(NULLIF(x.lot_id, ''), '_') || ':' || x.bidder_key || ':' || x.contract_ordinal, 't:' || x.unp, x.bidder_key, + NULLIF(TRIM(x.authority_name_raw), ''), x.display_native, COALESCE(x.currency, 'BGN'), x.contract_date, @@ -808,6 +811,7 @@ FROM ( END AS bidder_key FROM ( SELECT c.*, + c.authority_name AS authority_name_raw, CASE WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN COALESCE(c.current_value, c.signing_value) WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'BGN' THEN COALESCE(c.current_value, c.signing_value) / 1.95583 @@ -879,7 +883,7 @@ WHERE id IN ( ); INSERT OR IGNORE INTO contracts - (id, tender_id, bidder_id, amount, currency, signed_at, contract_number, signing_value, current_value, + (id, tender_id, bidder_id, ordering_unit_name, amount, currency, signed_at, contract_number, signing_value, current_value, annex_count, eu_funded, bids_received, contract_kind, awarded_to_group, value_flag, date_flag, amount_eur, fx_converted, fx_rate, signing_value_eur, current_value_eur, lot_id, document_number, published_at, contract_subject, @@ -892,6 +896,7 @@ SELECT COALESCE(NULLIF(x.lot_norm, ''), '_') || ':' || x.bidder_key || ':' || x.contract_ordinal, 't:' || x.unp, x.bidder_key, + NULLIF(TRIM(x.authority_name_raw), ''), x.display_native, COALESCE(x.currency, 'BGN'), x.contract_date, @@ -1056,6 +1061,7 @@ FROM ( END AS bidder_key FROM ( SELECT c.*, + c.authority_name AS authority_name_raw, CASE WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN COALESCE(c.current_value, c.signing_value) WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'BGN' THEN COALESCE(c.current_value, c.signing_value) / 1.95583 From 897a4213f0b5e1b48739c9b19272957d23e85e09 Mon Sep 17 00:00:00 2001 From: Todor Kolev Date: Sat, 18 Jul 2026 14:15:21 +0000 Subject: [PATCH 05/17] fix(etl,web): preserve identity-poor contracts via an unknown-bidder bucket MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bidder key fell to NULL when a contractor had an invalid-checksum ЕИК AND no name; `WHERE bidder_key IS NOT NULL` then silently dropped the whole contract. Never drop a contract for identity reasons — downgrade the key. - add a third key rung: invalid ЕИК + empty/NULL name -> one labelled `unknown:анонимен` bucket ("Неизвестен изпълнител", kind=unknown), so every previously-dropped contract stays in the corpus - compute the checksum + the three-rung key ONCE in a scratch `contractor_identity` table (NULL-safe raw-pair join), replacing the ~7 copy-pasted checksum blocks; delete the consistency test that pinned them - keep the bucket inside company_totals so money still reconciles; hide it only from browsable rankings (search_index, /companies, home top suppliers) - resolve the bucket's /companies slug so the contract-page link works - mirror all of it in refresh-slice; the checksum algorithm is byte-identical A new SQL test asserts no eligible contract is dropped and SUM(amount_eur) is preserved across both the normalize and refresh paths. --- .../db/src/contractor-identity-sql.test.ts | 104 +++++++ .../db/src/eik-checksum-consistency.test.ts | 33 --- packages/db/src/queries/companies.test.ts | 20 ++ packages/db/src/queries/companies.ts | 2 +- packages/db/src/queries/home.test.ts | 12 +- packages/db/src/queries/home.ts | 4 +- packages/db/src/queries/identity.test.ts | 4 + packages/db/src/queries/identity.ts | 4 +- scripts/normalize-raw.sql | 245 +++++++--------- scripts/precompute.sql | 3 +- scripts/refresh-slice.sql | 266 +++++++----------- 11 files changed, 347 insertions(+), 350 deletions(-) create mode 100644 packages/db/src/contractor-identity-sql.test.ts delete mode 100644 packages/db/src/eik-checksum-consistency.test.ts diff --git a/packages/db/src/contractor-identity-sql.test.ts b/packages/db/src/contractor-identity-sql.test.ts new file mode 100644 index 00000000..1bcfbb16 --- /dev/null +++ b/packages/db/src/contractor-identity-sql.test.ts @@ -0,0 +1,104 @@ +/// +import { readFileSync } from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { DatabaseSync } from 'node:sqlite'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); +const schema = readFileSync(resolve(root, 'packages/db/migrations/0000_init.sql'), 'utf8'); +const staging = readFileSync(resolve(root, 'scripts/work-staging-schema.sql'), 'utf8'); +const normalize = readFileSync(resolve(root, 'scripts/normalize-raw.sql'), 'utf8'); +const precompute = readFileSync(resolve(root, 'scripts/precompute.sql'), 'utf8'); +const refresh = readFileSync(resolve(root, 'scripts/refresh-slice.sql'), 'utf8'); + +const seed = ` +INSERT INTO raw_contracts + (source, fetched_at, unp, contract_number, contract_date, published_at, + authority_eik, authority_name, procurement_subject, estimated_value, + signing_value, currency, contractor_eik, contractor_name) +VALUES + ('eop:contracts:test', '2026-07-18T00:00:00Z', 'UNP-VALID', 'C-VALID', '2026-07-01', + '2026-07-02', '123456786', 'Тестов възложител', 'Валиден ЕИК', 100, 100, 'EUR', + '103267194', 'ТИПО ООД'), + ('eop:contracts:test', '2026-07-18T00:00:00Z', 'UNP-TYPO', 'C-TYPO', '2026-07-01', + '2026-07-02', '123456786', 'Тестов възложител', 'Грешна контролна цифра', 200, 200, 'EUR', + '103267195', 'ТИПО ООД'), + ('eop:contracts:test', '2026-07-18T00:00:00Z', 'UNP-EMPTY', 'C-EMPTY', '2026-07-01', + '2026-07-02', '123456786', 'Тестов възложител', 'Празно име', 300, 300, 'EUR', + '103267195', ''), + ('eop:contracts:test', '2026-07-18T00:00:00Z', 'UNP-NULL', 'C-NULL', '2026-07-01', + '2026-07-02', '123456786', 'Тестов възложител', 'Липсваща самоличност', 400, 400, 'EUR', + NULL, NULL); +`; + +function build(path: 'normalize' | 'refresh'): DatabaseSync { + const db = new DatabaseSync(':memory:'); + db.exec(schema); + db.exec(staging); + db.exec(seed); + if (path === 'normalize') { + db.exec(normalize); + db.exec(precompute); + } else { + db.exec(refresh); + } + return db; +} + +describe.each(['normalize', 'refresh'] as const)('%s contractor identity', (path) => { + it('preserves every eligible contract across valid, typo, empty, and NULL identities', () => { + const db = build(path); + try { + const contracts = db + .prepare('SELECT contract_number, bidder_id FROM contracts ORDER BY contract_number') + .all() as { contract_number: string; bidder_id: string }[]; + + expect(contracts).toEqual([ + { contract_number: 'C-EMPTY', bidder_id: 'unknown:анонимен' }, + { contract_number: 'C-NULL', bidder_id: 'unknown:анонимен' }, + { contract_number: 'C-TYPO', bidder_id: 'name:ТИПО ООД' }, + { contract_number: 'C-VALID', bidder_id: 'eik:103267194' }, + ]); + + expect( + db + .prepare( + `SELECT id, name, bulstat, eik_normalized, eik_valid, is_consortium, kind + FROM bidders WHERE id = 'unknown:анонимен'`, + ) + .get(), + ).toEqual({ + id: 'unknown:анонимен', + name: 'Неизвестен изпълнител', + bulstat: null, + eik_normalized: null, + eik_valid: 0, + is_consortium: 0, + kind: 'unknown', + }); + + const sums = db + .prepare( + `SELECT + (SELECT SUM(signing_value) FROM raw_contracts + WHERE source LIKE 'eop:%' OR source LIKE 'ocds:%') AS raw_total, + (SELECT SUM(amount_eur) FROM contracts) AS contract_total, + (SELECT SUM(won_eur) FROM company_totals) AS company_total`, + ) + .get() as { raw_total: number; contract_total: number; company_total: number }; + expect(sums).toEqual({ raw_total: 1000, contract_total: 1000, company_total: 1000 }); + + expect( + db + .prepare( + `SELECT COUNT(*) AS n FROM search_index + WHERE kind = 'company' AND ref = 'unknown:анонимен'`, + ) + .get(), + ).toEqual({ n: 0 }); + } finally { + db.close(); + } + }); +}); diff --git a/packages/db/src/eik-checksum-consistency.test.ts b/packages/db/src/eik-checksum-consistency.test.ts deleted file mode 100644 index 606cf4e0..00000000 --- a/packages/db/src/eik-checksum-consistency.test.ts +++ /dev/null @@ -1,33 +0,0 @@ -/// -import { readFileSync } from 'node:fs'; -import { dirname, resolve } from 'node:path'; -import { fileURLToPath } from 'node:url'; -import { describe, expect, it } from 'vitest'; - -const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); - -function checksumBlocks(path: string): string[] { - const sql = readFileSync(path, 'utf8'); - const pattern = - /^[ \t]*CASE\r?\n(?=[ \t]+WHEN eik_clean IS NULL\b[^\r\n]*LENGTH\(eik_clean\) NOT IN \(9, 13\) THEN 0$)[\s\S]*?^[ \t]*END AS eik_valid$/gm; - - return [...sql.matchAll(pattern)].map(([block]) => { - const lines = block.split(/\r?\n/); - const indentation = Math.min( - ...lines.filter((line) => line.trim()).map((line) => line.match(/^[ \t]*/)?.[0].length ?? 0), - ); - return lines.map((line) => line.slice(indentation)).join('\n'); - }); -} - -describe('ЕИК checksum copies stay consistent', () => { - it('keeps every checksum CASE block textually identical', () => { - const normalizeBlocks = checksumBlocks(resolve(root, 'scripts/normalize-raw.sql')); - const refreshBlocks = checksumBlocks(resolve(root, 'scripts/refresh-slice.sql')); - const blocks = [...normalizeBlocks, ...refreshBlocks]; - - expect(normalizeBlocks).toHaveLength(3); - expect(refreshBlocks).toHaveLength(4); - expect(new Set(blocks)).toEqual(new Set([blocks[0]])); - }); -}); diff --git a/packages/db/src/queries/companies.test.ts b/packages/db/src/queries/companies.test.ts index a4643d02..b0e9b9f0 100644 --- a/packages/db/src/queries/companies.test.ts +++ b/packages/db/src/queries/companies.test.ts @@ -105,6 +105,26 @@ describe('streamCompaniesCsv', () => { expect(countRows(filtered)).toBeLessThan(countRows(unfiltered)); expect(countRows(filtered)).toBe(1); }); + + it('excludes the unknown identity bucket from the list and CSV', async () => { + const db = fakeDb(); + const sql: string[] = []; + const real = db.prepare.bind(db); + db.prepare = ((query: string) => { + sql.push(query); + return real(query); + }) as typeof db.prepare; + + await listCompanies(db, {}); + await streamCompaniesCsv(db, {}).text(); + + expect(sql.filter((query) => query.includes('company_totals'))).not.toHaveLength(0); + expect( + sql.filter((query) => query.includes('company_totals')).every((query) => + query.includes("kind <> 'unknown'"), + ), + ).toBe(true); + }); }); describe('prototype-key params (untrusted query values)', () => { diff --git a/packages/db/src/queries/companies.ts b/packages/db/src/queries/companies.ts index 70a1fa40..8f266523 100644 --- a/packages/db/src/queries/companies.ts +++ b/packages/db/src/queries/companies.ts @@ -109,7 +109,7 @@ function source(p: CompanyListParams): { from: string; params: unknown[] } { * Keep consumed filter keys in sync with COMPANY_FILTER_KEYS and companyFilterSignature(). */ function entityWhere(p: CompanyListParams): { sql: string; params: unknown[] } { - const where: string[] = []; + const where: string[] = [`kind <> 'unknown'`]; const params: unknown[] = []; if (p.kinds?.length && p.kinds.length < 2) { where.push(`kind = ?`); diff --git a/packages/db/src/queries/home.test.ts b/packages/db/src/queries/home.test.ts index 0302eb45..257c8be5 100644 --- a/packages/db/src/queries/home.test.ts +++ b/packages/db/src/queries/home.test.ts @@ -61,9 +61,10 @@ const totalsRow = { refreshed_at: '2024-06-02T10:00:00Z', }; -function fakeDb(totals: typeof totalsRow | null): D1Database { +function fakeDb(totals: typeof totalsRow | null, capture?: string[]): D1Database { return { prepare(sql: string) { + capture?.push(sql); return { bind() { return this; @@ -121,6 +122,15 @@ describe('getHomeData', () => { expect(data.topMunicipalities).toHaveLength(1); }); + it('excludes the unknown identity bucket from top companies', async () => { + const sql: string[] = []; + await getHomeData(fakeDb(totalsRow, sql)); + + expect(sql.find((query) => query.includes('FROM company_totals'))).toContain( + "WHERE kind <> 'unknown'", + ); + }); + it('includes single-offer contract lists', async () => { const data = await getHomeData(fakeDb(totalsRow)); diff --git a/packages/db/src/queries/home.ts b/packages/db/src/queries/home.ts index fe588134..41fc3ff3 100644 --- a/packages/db/src/queries/home.ts +++ b/packages/db/src/queries/home.ts @@ -53,7 +53,9 @@ export async function getHomeData(db: D1Database): Promise { const [companies, ministries, municipalities, recentSingleOffer, topSingleOffer, singleOfferRow] = await Promise.all([ db - .prepare(`SELECT * FROM company_totals ORDER BY won_eur DESC, bidder_id LIMIT 10`) + .prepare( + `SELECT * FROM company_totals WHERE kind <> 'unknown' ORDER BY won_eur DESC, bidder_id LIMIT 10`, + ) .all(), db .prepare( diff --git a/packages/db/src/queries/identity.test.ts b/packages/db/src/queries/identity.test.ts index 2acc6ccf..7ca09ed7 100644 --- a/packages/db/src/queries/identity.test.ts +++ b/packages/db/src/queries/identity.test.ts @@ -25,6 +25,10 @@ describe('company slug', () => { it('round-trips a 13-digit ЕИК', () => { expect(bidderIdFromSlug(companySlug('eik:8316417910124'))).toBe('eik:8316417910124'); }); + it('resolves the unknown bidder linked from contract pages', () => { + expect(companySlug('unknown:анонимен')).toBe('unknown:анонимен'); + expect(bidderIdFromSlug('unknown:анонимен')).toBe('unknown:анонимен'); + }); }); describe('authority / contract slugs', () => { diff --git a/packages/db/src/queries/identity.ts b/packages/db/src/queries/identity.ts index a12067b3..2780bfc5 100644 --- a/packages/db/src/queries/identity.ts +++ b/packages/db/src/queries/identity.ts @@ -21,7 +21,8 @@ function b64urlDecode(s: string): string { const EIK_RE = /^\d{9}(\d{4})?$/; -/** bidder id → `/companies/:slug` segment. Valid ЕИК → the digits; name-keyed → `n` + base64url(name). */ +/** bidder id → `/companies/:slug` segment. Valid ЕИК → digits; name-keyed → `n` + base64url(name); + * the labelled unknown bucket stays literal so contract-page links remain resolvable. */ export function companySlug(bidderId: string): string { if (bidderId.startsWith('eik:')) return bidderId.slice(4); if (bidderId.startsWith('name:')) return 'n' + b64urlEncode(bidderId.slice(5)); @@ -31,6 +32,7 @@ export function companySlug(bidderId: string): string { /** `/companies/:slug` segment → bidder id, or null if it cannot be decoded. */ export function bidderIdFromSlug(slug: string): string | null { if (EIK_RE.test(slug)) return 'eik:' + slug; + if (slug === 'unknown:анонимен') return slug; if (slug.startsWith('n')) { try { return 'name:' + b64urlDecode(slug.slice(1)); diff --git a/scripts/normalize-raw.sql b/scripts/normalize-raw.sql index 74ce5bf4..c72a5f2a 100644 --- a/scripts/normalize-raw.sql +++ b/scripts/normalize-raw.sql @@ -22,8 +22,8 @@ -- * Currency is kept per row as it appears (BGN pre-2026, EUR from 2026, a few foreign) — -- NOT coerced to one currency. Money sums must group/convert by currency downstream. -- * Authorities dedupe on ЕИК (Вид на възложителя kept as `type`); a canonical name is kept. --- * Bidders dedupe on raw contractor ЕИК (kept verbatim in bulstat); is_consortium comes from --- the admin "възложена на група" flag (awarded_to_group), not a name heuristic. Resolving the +-- * Bidders use a checksum-valid contractor ЕИК, then normalised name, then one unknown bucket; +-- is_consortium comes from the entity name, while awarded_to_group stays contract-scoped. Resolving the -- members hidden behind a single consortium ЕИК needs the Търговски регистър (joined on ЕИК), -- a parked future pipeline; for now the full value is attributed to the consortium entity. -- * Tenders come from the tenders-export header row (one per УНП); lots from its lot rows. @@ -229,83 +229,100 @@ FROM raw_tenders t WHERE t.lot_id IS NOT NULL AND EXISTS (SELECT 1 FROM tenders te WHERE te.id = 't:' || t.unp); --- 4) Bidders — winning contractors, with a robust identity key: --- * checksum-valid Bulgarian ЕИК → key by ЕИК ('eik:') --- * otherwise (withheld 'не се публикува', foreign id, multi-ЕИК consortium, missing) → --- key by NORMALISED NAME ('name:') --- Keying invalid ЕИК by name stops the collapse where ~595 distinct withheld-ЕИК contractors --- merged onto one node. bulstat/eik_normalized stay set only for a valid ЕИК (NULL otherwise, --- which keeps the bulstat UNIQUE happy). is_consortium describes the ENTITY (a JV), so it is --- name-based — a semicolon member list or ДЗЗД / ОБЕДИНЕНИЕ / КОНСОРЦИУМ in the name; the --- per-contract awarded_to_group flag lives on contracts, not here. -INSERT OR IGNORE INTO bidders (id, name, bulstat, eik_normalized, eik_valid, is_consortium, kind) +-- 4) Winning-contractor identity — derive the checksum and three-rung key exactly once, then use +-- the NULL-safe raw pair for every bidder/contract projection below. +DROP TABLE IF EXISTS contractor_identity; +CREATE TABLE contractor_identity ( + eik_raw TEXT, + name_raw TEXT, + eik_clean TEXT, + eik_valid INTEGER NOT NULL, + bidder_key TEXT NOT NULL, + PRIMARY KEY (eik_raw, name_raw) +); + +INSERT INTO contractor_identity (eik_raw, name_raw, eik_clean, eik_valid, bidder_key) SELECT - bidder_key, - MIN(contractor_name), - MIN(CASE WHEN eik_valid = 1 THEN eik_clean END), - MIN(CASE WHEN eik_valid = 1 THEN eik_clean END), - MAX(eik_valid), - MAX(grp), - CASE WHEN MAX(grp) = 1 THEN 'consortium' ELSE 'company' END + eik_raw, + name_raw, + eik_clean, + eik_valid, + CASE + WHEN eik_valid = 1 THEN 'eik:' || eik_clean + WHEN name_raw IS NOT NULL AND TRIM(name_raw) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(name_raw, ' ', ' '), ' ', ' '))) + ELSE 'unknown:анонимен' + END FROM ( SELECT - contractor_name, + eik_raw, + name_raw, eik_clean, - eik_valid, - CASE - WHEN eik_valid = 1 THEN 'eik:' || eik_clean - WHEN contractor_name IS NOT NULL AND TRIM(contractor_name) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(contractor_name, ' ', ' '), ' ', ' '))) - ELSE NULL - END AS bidder_key, + -- Bulgarian ЕИК/Булстат control-digit (checksum) validation. A merely SYNTACTIC 9/13-digit + -- check let the fake/service code „000000001" and wrong-digit typo twins pass, collapsing + -- unrelated foreign suppliers (Elsevier + Clarivate/Web of Science + a gas consultancy + a + -- 102 EUR construction line) onto one node (#195). Enforce the real checksum so invalid codes + -- get eik_valid = 0 and fall back to the name-based key, which splits them apart again. + -- 9-digit: weight positions 1..8 by 1..8; control = sum % 11. If that is 10, re-weight by + -- 3..10; a second 10 → 0. The 9th digit must equal the control. + -- 13-digit: the leading 9 digits must themselves be a valid 9-digit ЕИК, then weight + -- positions 9..12 by 2,7,3,5 (fallback 4,9,5,7; second 10 → 0). The 13th digit + -- must equal that control. CASE - WHEN contractor_name LIKE '%;%' - OR UPPER(contractor_name) LIKE '%ДЗЗД%' - OR UPPER(contractor_name) LIKE '%ОБЕДИНЕНИЕ%' - OR UPPER(contractor_name) LIKE '%КОНСОРЦИУМ%' - THEN 1 ELSE 0 - END AS grp + WHEN eik_clean IS NULL OR eik_clean IN ('000000000', '0000000000000') OR eik_clean GLOB '*[^0-9]*' OR LENGTH(eik_clean) NOT IN (9, 13) THEN 0 + WHEN ( + CASE + WHEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 + WHEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 + ELSE 0 + END + ) <> CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) THEN 0 + WHEN LENGTH(eik_clean) = 9 THEN 1 + WHEN ( + CASE + WHEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 + WHEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 + ELSE 0 + END + ) = CAST(SUBSTR(eik_clean, 13, 1) AS INTEGER) THEN 1 + ELSE 0 + END AS eik_valid FROM ( SELECT - contractor_name, - eik_clean, - -- Bulgarian ЕИК/Булстат control-digit (checksum) validation. A merely SYNTACTIC 9/13-digit - -- check let the fake/service code „000000001" and wrong-digit typo twins pass, collapsing - -- unrelated foreign suppliers (Elsevier + Clarivate/Web of Science + a gas consultancy + a - -- 102 EUR construction line) onto one node (#195). Enforce the real checksum so invalid codes - -- get eik_valid = 0 and fall back to the name-based key, which splits them apart again. - -- 9-digit: weight positions 1..8 by 1..8; control = sum % 11. If that is 10, re-weight by - -- 3..10; a second 10 → 0. The 9th digit must equal the control. - -- 13-digit: the leading 9 digits must themselves be a valid 9-digit ЕИК, then weight - -- positions 9..12 by 2,7,3,5 (fallback 4,9,5,7; second 10 → 0). The 13th digit - -- must equal that control. - CASE - WHEN eik_clean IS NULL OR eik_clean IN ('000000000', '0000000000000') OR eik_clean GLOB '*[^0-9]*' OR LENGTH(eik_clean) NOT IN (9, 13) THEN 0 - WHEN ( - CASE - WHEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 - WHEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 - ELSE 0 - END - ) <> CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) THEN 0 - WHEN LENGTH(eik_clean) = 9 THEN 1 - WHEN ( - CASE - WHEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 - WHEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 - ELSE 0 - END - ) = CAST(SUBSTR(eik_clean, 13, 1) AS INTEGER) THEN 1 - ELSE 0 - END AS eik_valid - FROM ( - SELECT - contractor_name, - TRIM(CASE WHEN contractor_eik LIKE 'ЕИК %' THEN SUBSTR(contractor_eik, 5) ELSE contractor_eik END) AS eik_clean - FROM raw_contracts WHERE source LIKE 'eop:%' OR source LIKE 'ocds:%' - ) + contractor_eik AS eik_raw, + contractor_name AS name_raw, + TRIM(CASE WHEN contractor_eik LIKE 'ЕИК %' THEN SUBSTR(contractor_eik, 5) ELSE contractor_eik END) AS eik_clean + FROM (SELECT DISTINCT contractor_eik, contractor_name FROM raw_contracts WHERE source LIKE 'eop:%' OR source LIKE 'ocds:%') ) -) -WHERE bidder_key IS NOT NULL +); + +-- 4a) Bidders — valid ЕИК first, then normalised name, then one labelled unknown bucket. The +-- bucket keeps identity-poor contracts attached without polluting bulstat/eik_normalized. +INSERT OR IGNORE INTO bidders (id, name, bulstat, eik_normalized, eik_valid, is_consortium, kind) +SELECT + bidder_key, + CASE WHEN bidder_key = 'unknown:анонимен' THEN 'Неизвестен изпълнител' ELSE MIN(name_raw) END, + MIN(CASE WHEN eik_valid = 1 THEN eik_clean END), + MIN(CASE WHEN eik_valid = 1 THEN eik_clean END), + MAX(eik_valid), + MAX(CASE + WHEN name_raw LIKE '%;%' + OR UPPER(name_raw) LIKE '%ДЗЗД%' + OR UPPER(name_raw) LIKE '%ОБЕДИНЕНИЕ%' + OR UPPER(name_raw) LIKE '%КОНСОРЦИУМ%' + THEN 1 ELSE 0 + END), + CASE + WHEN bidder_key = 'unknown:анонимен' THEN 'unknown' + WHEN MAX(CASE + WHEN name_raw LIKE '%;%' + OR UPPER(name_raw) LIKE '%ДЗЗД%' + OR UPPER(name_raw) LIKE '%ОБЕДИНЕНИЕ%' + OR UPPER(name_raw) LIKE '%КОНСОРЦИУМ%' + THEN 1 ELSE 0 + END) = 1 THEN 'consortium' + ELSE 'company' + END +FROM contractor_identity GROUP BY bidder_key; -- 4b) Curated public-owned winner classification. Exact EIK matches cover the allowlist; a small @@ -523,42 +540,9 @@ FROM ( AND c.contract_date > date(c.published_at, '+2 day') THEN 'signed_after_publication' ELSE 'ok' END AS date_flag, - ( - SELECT - CASE - WHEN eik_valid = 1 THEN 'eik:' || eik_clean - WHEN c.contractor_name IS NOT NULL AND TRIM(c.contractor_name) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(c.contractor_name, ' ', ' '), ' ', ' '))) - ELSE NULL - END - FROM ( - SELECT - eik_clean, - CASE - WHEN eik_clean IS NULL OR eik_clean IN ('000000000', '0000000000000') OR eik_clean GLOB '*[^0-9]*' OR LENGTH(eik_clean) NOT IN (9, 13) THEN 0 - WHEN ( - CASE - WHEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 - WHEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 - ELSE 0 - END - ) <> CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) THEN 0 - WHEN LENGTH(eik_clean) = 9 THEN 1 - WHEN ( - CASE - WHEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 - WHEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 - ELSE 0 - END - ) = CAST(SUBSTR(eik_clean, 13, 1) AS INTEGER) THEN 1 - ELSE 0 - END AS eik_valid - FROM ( - SELECT TRIM(CASE WHEN c.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(c.contractor_eik, 5) ELSE c.contractor_eik END) AS eik_clean - ) - ) - ) AS bidder_key + c.bidder_key FROM ( - SELECT c.*, + SELECT c.*, ci.bidder_key, CASE WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN COALESCE(c.current_value, c.signing_value) WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'BGN' THEN COALESCE(c.current_value, c.signing_value) / 1.95583 @@ -588,6 +572,8 @@ FROM ( END AS proc_est_eur, t.estimated_value AS proc_est_native FROM raw_contracts c + JOIN contractor_identity ci + ON c.contractor_eik IS ci.eik_raw AND c.contractor_name IS ci.name_raw LEFT JOIN tenders t ON t.id = 't:' || c.unp ) c -- EOP always; an OCDS row only when no EOP row shares its contract_number - EOP wins. @@ -616,8 +602,7 @@ FROM ( ) z ) y ) x -WHERE x.bidder_key IS NOT NULL - AND x.display_native IS NOT NULL +WHERE x.display_native IS NOT NULL AND EXISTS (SELECT 1 FROM tenders te WHERE te.id = 't:' || x.unp) AND EXISTS (SELECT 1 FROM bidders b WHERE b.id = x.bidder_key); @@ -818,42 +803,9 @@ SELECT 1, WHEN c.proc_est_eur > 0 AND c.eff_eur >= 10 * c.proc_est_eur THEN 'review' ELSE 'ok' END AS value_flag, - ( - SELECT - CASE - WHEN eik_valid = 1 THEN 'eik:' || eik_clean - WHEN c.contractor_name IS NOT NULL AND TRIM(c.contractor_name) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(c.contractor_name, ' ', ' '), ' ', ' '))) - ELSE NULL - END - FROM ( - SELECT - eik_clean, - CASE - WHEN eik_clean IS NULL OR eik_clean IN ('000000000', '0000000000000') OR eik_clean GLOB '*[^0-9]*' OR LENGTH(eik_clean) NOT IN (9, 13) THEN 0 - WHEN ( - CASE - WHEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 - WHEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 - ELSE 0 - END - ) <> CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) THEN 0 - WHEN LENGTH(eik_clean) = 9 THEN 1 - WHEN ( - CASE - WHEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 - WHEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 - ELSE 0 - END - ) = CAST(SUBSTR(eik_clean, 13, 1) AS INTEGER) THEN 1 - ELSE 0 - END AS eik_valid - FROM ( - SELECT TRIM(CASE WHEN c.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(c.contractor_eik, 5) ELSE c.contractor_eik END) AS eik_clean - ) - ) - ) AS bidder_key + c.bidder_key FROM ( - SELECT c.*, + SELECT c.*, ci.bidder_key, CASE WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN COALESCE(c.current_value, c.signing_value) WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'BGN' THEN COALESCE(c.current_value, c.signing_value) / 1.95583 @@ -883,6 +835,8 @@ SELECT 1, END AS proc_est_eur, t.estimated_value AS proc_est_native FROM raw_contracts c + JOIN contractor_identity ci + ON c.contractor_eik IS ci.eik_raw AND c.contractor_name IS ci.name_raw LEFT JOIN tenders t ON t.id = 't:' || c.unp ) c -- Eligibility must mirror the INSERT INTO contracts WHERE exactly, so this candidate count is a @@ -895,8 +849,7 @@ SELECT 1, SELECT 1 FROM raw_contracts a WHERE a.source LIKE 'eop:%' AND a.contract_number = c.contract_number)) ) c - WHERE c.bidder_key IS NOT NULL - AND CASE c.value_flag + WHERE CASE c.value_flag WHEN 'annex_suspect' THEN COALESCE(c.signing_value, c.current_value) ELSE COALESCE(c.current_value, c.signing_value) END IS NOT NULL @@ -906,6 +859,8 @@ SELECT 1, (SELECT COUNT(*) FROM contracts), datetime('now'); +DROP TABLE contractor_identity; + -- Summary (last result set printed by `wrangler d1 execute`) SELECT (SELECT COUNT(*) FROM authorities) AS authorities, diff --git a/scripts/precompute.sql b/scripts/precompute.sql index d52642d1..ce27ead1 100644 --- a/scripts/precompute.sql +++ b/scripts/precompute.sql @@ -158,7 +158,8 @@ SELECT 'authority', at.authority_id, at.name, COALESCE(substr(at.authority_id, 6 FROM authority_totals at; INSERT INTO search_index (kind, ref, title, ident, subtitle, amount) SELECT 'company', ct.bidder_id, ct.name, COALESCE(ct.eik, ''), COALESCE(ct.settlement, ''), ct.won_eur -FROM company_totals ct; +FROM company_totals ct +WHERE ct.bidder_id <> 'unknown:анонимен'; INSERT INTO search_index (kind, ref, title, ident, subtitle, amount) SELECT 'contract', c.id, COALESCE(NULLIF(c.contract_subject, ''), t.title), COALESCE(t.source_id, ''), diff --git a/scripts/refresh-slice.sql b/scripts/refresh-slice.sql index a145895c..4c383499 100644 --- a/scripts/refresh-slice.sql +++ b/scripts/refresh-slice.sql @@ -45,61 +45,97 @@ UPDATE authorities SET type_group = CASE END WHERE type_group IS NULL; --- ── 2) Bidders referenced by OCDS staging (new ones only) — same identity rule as normalize step 4 ── +-- ── 2) Winning-contractor identity — derive the checksum and three-rung key once for this slice. ── +DROP TABLE IF EXISTS contractor_identity; +CREATE TABLE contractor_identity ( + eik_raw TEXT, + name_raw TEXT, + eik_clean TEXT, + eik_valid INTEGER NOT NULL, + bidder_key TEXT NOT NULL, + PRIMARY KEY (eik_raw, name_raw) +); + +INSERT INTO contractor_identity (eik_raw, name_raw, eik_clean, eik_valid, bidder_key) +SELECT + eik_raw, + name_raw, + eik_clean, + eik_valid, + CASE + WHEN eik_valid = 1 THEN 'eik:' || eik_clean + WHEN name_raw IS NOT NULL AND TRIM(name_raw) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(name_raw, ' ', ' '), ' ', ' '))) + ELSE 'unknown:анонимен' + END +FROM ( + SELECT + eik_raw, + name_raw, + eik_clean, + -- Bulgarian ЕИК/Булстат control-digit (checksum) validation. A merely SYNTACTIC 9/13-digit + -- check let the fake/service code „000000001" and wrong-digit typo twins pass, collapsing + -- unrelated foreign suppliers (Elsevier + Clarivate/Web of Science + a gas consultancy + a + -- 102 EUR construction line) onto one node (#195). Enforce the real checksum so invalid codes + -- get eik_valid = 0 and fall back to the name-based key, which splits them apart again. + -- 9-digit: weight positions 1..8 by 1..8; control = sum % 11. If that is 10, re-weight by + -- 3..10; a second 10 → 0. The 9th digit must equal the control. + -- 13-digit: the leading 9 digits must themselves be a valid 9-digit ЕИК, then weight + -- positions 9..12 by 2,7,3,5 (fallback 4,9,5,7; second 10 → 0). The 13th digit + -- must equal that control. + CASE + WHEN eik_clean IS NULL OR eik_clean IN ('000000000', '0000000000000') OR eik_clean GLOB '*[^0-9]*' OR LENGTH(eik_clean) NOT IN (9, 13) THEN 0 + WHEN ( + CASE + WHEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 + WHEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 + ELSE 0 + END + ) <> CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) THEN 0 + WHEN LENGTH(eik_clean) = 9 THEN 1 + WHEN ( + CASE + WHEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 + WHEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 + ELSE 0 + END + ) = CAST(SUBSTR(eik_clean, 13, 1) AS INTEGER) THEN 1 + ELSE 0 + END AS eik_valid + FROM ( + SELECT + contractor_eik AS eik_raw, + contractor_name AS name_raw, + TRIM(CASE WHEN contractor_eik LIKE 'ЕИК %' THEN SUBSTR(contractor_eik, 5) ELSE contractor_eik END) AS eik_clean + FROM (SELECT DISTINCT contractor_eik, contractor_name FROM raw_contracts WHERE source LIKE 'eop:%' OR source LIKE 'ocds:%') + ) +); + INSERT OR IGNORE INTO bidders (id, name, bulstat, eik_normalized, eik_valid, is_consortium, kind) SELECT bidder_key, - MIN(contractor_name), + CASE WHEN bidder_key = 'unknown:анонимен' THEN 'Неизвестен изпълнител' ELSE MIN(name_raw) END, MIN(CASE WHEN eik_valid = 1 THEN eik_clean END), MIN(CASE WHEN eik_valid = 1 THEN eik_clean END), MAX(eik_valid), - MAX(grp), - CASE WHEN MAX(grp) = 1 THEN 'consortium' ELSE 'company' END -FROM ( - SELECT contractor_name, eik_clean, eik_valid, - CASE - WHEN eik_valid = 1 THEN 'eik:' || eik_clean - WHEN contractor_name IS NOT NULL AND TRIM(contractor_name) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(contractor_name, ' ', ' '), ' ', ' '))) - ELSE NULL - END AS bidder_key, - CASE - WHEN contractor_name LIKE '%;%' - OR UPPER(contractor_name) LIKE '%ДЗЗД%' - OR UPPER(contractor_name) LIKE '%ОБЕДИНЕНИЕ%' - OR UPPER(contractor_name) LIKE '%КОНСОРЦИУМ%' + MAX(CASE + WHEN name_raw LIKE '%;%' + OR UPPER(name_raw) LIKE '%ДЗЗД%' + OR UPPER(name_raw) LIKE '%ОБЕДИНЕНИЕ%' + OR UPPER(name_raw) LIKE '%КОНСОРЦИУМ%' + THEN 1 ELSE 0 + END), + CASE + WHEN bidder_key = 'unknown:анонимен' THEN 'unknown' + WHEN MAX(CASE + WHEN name_raw LIKE '%;%' + OR UPPER(name_raw) LIKE '%ДЗЗД%' + OR UPPER(name_raw) LIKE '%ОБЕДИНЕНИЕ%' + OR UPPER(name_raw) LIKE '%КОНСОРЦИУМ%' THEN 1 ELSE 0 - END AS grp - FROM ( - SELECT contractor_name, eik_clean, - -- Bulgarian ЕИК/Булстат checksum — identical to normalize-raw.sql step 4 (#195), so the - -- incremental path's bidder identity stays in parity with the full rebuild. - CASE - WHEN eik_clean IS NULL OR eik_clean IN ('000000000', '0000000000000') OR eik_clean GLOB '*[^0-9]*' OR LENGTH(eik_clean) NOT IN (9, 13) THEN 0 - WHEN ( - CASE - WHEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 - WHEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 - ELSE 0 - END - ) <> CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) THEN 0 - WHEN LENGTH(eik_clean) = 9 THEN 1 - WHEN ( - CASE - WHEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 - WHEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 - ELSE 0 - END - ) = CAST(SUBSTR(eik_clean, 13, 1) AS INTEGER) THEN 1 - ELSE 0 - END AS eik_valid - FROM ( - SELECT contractor_name, - TRIM(CASE WHEN contractor_eik LIKE 'ЕИК %' THEN SUBSTR(contractor_eik, 5) ELSE contractor_eik END) AS eik_clean - FROM raw_contracts WHERE source LIKE 'eop:%' OR source LIKE 'ocds:%' - ) - ) -) -WHERE bidder_key IS NOT NULL + END) = 1 THEN 'consortium' + ELSE 'company' + END +FROM contractor_identity GROUP BY bidder_key ON CONFLICT(id) DO UPDATE SET name = CASE @@ -702,42 +738,9 @@ FROM ( AND c.contract_date > date(c.published_at, '+2 day') THEN 'signed_after_publication' ELSE 'ok' END AS date_flag, - ( - SELECT - CASE - WHEN eik_valid = 1 THEN 'eik:' || eik_clean - WHEN c.contractor_name IS NOT NULL AND TRIM(c.contractor_name) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(c.contractor_name, ' ', ' '), ' ', ' '))) - ELSE NULL - END - FROM ( - SELECT - eik_clean, - CASE - WHEN eik_clean IS NULL OR eik_clean IN ('000000000', '0000000000000') OR eik_clean GLOB '*[^0-9]*' OR LENGTH(eik_clean) NOT IN (9, 13) THEN 0 - WHEN ( - CASE - WHEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 - WHEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 - ELSE 0 - END - ) <> CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) THEN 0 - WHEN LENGTH(eik_clean) = 9 THEN 1 - WHEN ( - CASE - WHEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 - WHEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 - ELSE 0 - END - ) = CAST(SUBSTR(eik_clean, 13, 1) AS INTEGER) THEN 1 - ELSE 0 - END AS eik_valid - FROM ( - SELECT TRIM(CASE WHEN c.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(c.contractor_eik, 5) ELSE c.contractor_eik END) AS eik_clean - ) - ) - ) AS bidder_key + c.bidder_key FROM ( - SELECT c.*, + SELECT c.*, ci.bidder_key, CASE WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN COALESCE(c.current_value, c.signing_value) WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'BGN' THEN COALESCE(c.current_value, c.signing_value) / 1.95583 @@ -767,6 +770,8 @@ FROM ( END AS proc_est_eur, t.estimated_value AS proc_est_native FROM raw_contracts c + JOIN contractor_identity ci + ON c.contractor_eik IS ci.eik_raw AND c.contractor_name IS ci.name_raw LEFT JOIN tenders t ON t.id = 't:' || c.unp ) c WHERE c.source LIKE 'ocds:%' @@ -774,8 +779,7 @@ FROM ( ) y ) q ) x -WHERE x.bidder_key IS NOT NULL - AND x.display_native IS NOT NULL +WHERE x.display_native IS NOT NULL AND EXISTS (SELECT 1 FROM tenders te WHERE te.id = 't:' || x.unp) AND EXISTS (SELECT 1 FROM bidders b WHERE b.id = x.bidder_key) -- EOP wins: skip OCDS rows when the transient window has an EOP row for the same document. @@ -977,42 +981,9 @@ FROM ( AND c.contract_date > date(c.published_at, '+2 day') THEN 'signed_after_publication' ELSE 'ok' END AS date_flag, - ( - SELECT - CASE - WHEN eik_valid = 1 THEN 'eik:' || eik_clean - WHEN c.contractor_name IS NOT NULL AND TRIM(c.contractor_name) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(c.contractor_name, ' ', ' '), ' ', ' '))) - ELSE NULL - END - FROM ( - SELECT - eik_clean, - CASE - WHEN eik_clean IS NULL OR eik_clean IN ('000000000', '0000000000000') OR eik_clean GLOB '*[^0-9]*' OR LENGTH(eik_clean) NOT IN (9, 13) THEN 0 - WHEN ( - CASE - WHEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 - WHEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 - ELSE 0 - END - ) <> CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) THEN 0 - WHEN LENGTH(eik_clean) = 9 THEN 1 - WHEN ( - CASE - WHEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 - WHEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 - ELSE 0 - END - ) = CAST(SUBSTR(eik_clean, 13, 1) AS INTEGER) THEN 1 - ELSE 0 - END AS eik_valid - FROM ( - SELECT TRIM(CASE WHEN c.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(c.contractor_eik, 5) ELSE c.contractor_eik END) AS eik_clean - ) - ) - ) AS bidder_key + c.bidder_key FROM ( - SELECT c.*, + SELECT c.*, ci.bidder_key, CASE WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN COALESCE(c.current_value, c.signing_value) WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'BGN' THEN COALESCE(c.current_value, c.signing_value) / 1.95583 @@ -1042,6 +1013,8 @@ FROM ( END AS proc_est_eur, t.estimated_value AS proc_est_native FROM raw_contracts c + JOIN contractor_identity ci + ON c.contractor_eik IS ci.eik_raw AND c.contractor_name IS ci.name_raw LEFT JOIN tenders t ON t.id = 't:' || c.unp ) c WHERE c.source LIKE 'eop:%' @@ -1061,8 +1034,7 @@ FROM ( ) y ) q ) x -WHERE x.bidder_key IS NOT NULL - AND x.display_native IS NOT NULL +WHERE x.display_native IS NOT NULL AND EXISTS (SELECT 1 FROM tenders te WHERE te.id = 't:' || x.unp) AND EXISTS (SELECT 1 FROM bidders b WHERE b.id = x.bidder_key) AND NOT EXISTS ( @@ -1318,50 +1290,9 @@ INSERT OR IGNORE INTO refresh_touched_bidders (bidder_id) SELECT b.id FROM bidders b WHERE b.eik_normalized IN (SELECT eik FROM raw_ocds_parties WHERE eik IS NOT NULL) - OR b.id IN ( - SELECT bidder_key - FROM ( - SELECT - contractor_name, - CASE - WHEN eik_valid = 1 THEN 'eik:' || eik_clean - WHEN contractor_name IS NOT NULL AND TRIM(contractor_name) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(contractor_name, ' ', ' '), ' ', ' '))) - ELSE NULL - END AS bidder_key - FROM ( - SELECT - contractor_name, - eik_clean, - -- Same Bulgarian ЕИК checksum as the bidder INSERT above and normalize-raw.sql (#195), - -- so the touched-bidder key matches the actual bidders.id keying. - CASE - WHEN eik_clean IS NULL OR eik_clean IN ('000000000', '0000000000000') OR eik_clean GLOB '*[^0-9]*' OR LENGTH(eik_clean) NOT IN (9, 13) THEN 0 - WHEN ( - CASE - WHEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 - WHEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 - ELSE 0 - END - ) <> CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) THEN 0 - WHEN LENGTH(eik_clean) = 9 THEN 1 - WHEN ( - CASE - WHEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 - WHEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 - ELSE 0 - END - ) = CAST(SUBSTR(eik_clean, 13, 1) AS INTEGER) THEN 1 - ELSE 0 - END AS eik_valid - FROM ( - SELECT contractor_name, - TRIM(CASE WHEN contractor_eik LIKE 'ЕИК %' THEN SUBSTR(contractor_eik, 5) ELSE contractor_eik END) AS eik_clean - FROM raw_contracts WHERE source LIKE 'eop:%' OR source LIKE 'ocds:%' - ) - ) - ) - WHERE bidder_key IS NOT NULL - ); + OR b.id IN (SELECT bidder_key FROM contractor_identity); + +DROP TABLE contractor_identity; -- 6) Refresh rollups + FTS. Only the D1-hot rollups are scoped to touched rows; cheaper rollups stay -- full-recomputed in isolated batches so convergence stays simple. @@ -1407,7 +1338,8 @@ GROUP BY t.authority_id, c.bidder_id; DELETE FROM search_index WHERE kind = 'company'; INSERT INTO search_index (kind, ref, title, ident, subtitle, amount) SELECT 'company', ct.bidder_id, ct.name, COALESCE(ct.eik, ''), COALESCE(ct.settlement, ''), ct.won_eur -FROM company_totals ct; +FROM company_totals ct +WHERE ct.bidder_id <> 'unknown:анонимен'; DELETE FROM search_index WHERE kind = 'authority'; INSERT INTO search_index (kind, ref, title, ident, subtitle, amount) SELECT 'authority', at.authority_id, at.name, COALESCE(substr(at.authority_id, 6), ''), COALESCE(at.settlement, ''), at.spent_eur From efb1091f187eaeca72829dbd3902b88ab2c2f64d Mon Sep 17 00:00:00 2001 From: Todor Kolev Date: Sun, 19 Jul 2026 12:36:27 +0000 Subject: [PATCH 06/17] fix(etl): fold contractor name-key over case, quotes and dash variants MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Name-keyed bidders (no valid ЕИК) split the same company across surface variants because SQLite UPPER() is ASCII-only and quote/dash encodings vary. Normalise the name in the `contractor_identity` scratch table so Cyrillic case, quote styles and dash encodings collapse to one key — while dash vs space stays distinct, so genuinely different names are not over-merged. - staged `name_norm` UPDATEs (whitespace + quotes + dashes + Cyrillic case), each a shallow expression: a single ~60-deep nested REPLACE overflows the sqlite3 CLI parser (v3.40.1) on the import path, so the fold is split into stages of <=15 REPLACE calls - the fold is byte-identical in normalize-raw and refresh-slice - tests assert quote/case/space variants merge to one key, en-dash == hyphen, and hyphen != space (over-fold guard); SUM(amount_eur) still reconciles --- .../db/src/contractor-identity-sql.test.ts | 32 +++++++++++++++++-- scripts/normalize-raw.sql | 32 +++++++++++++++++-- scripts/refresh-slice.sql | 32 +++++++++++++++++-- 3 files changed, 88 insertions(+), 8 deletions(-) diff --git a/packages/db/src/contractor-identity-sql.test.ts b/packages/db/src/contractor-identity-sql.test.ts index 1bcfbb16..13aa10ea 100644 --- a/packages/db/src/contractor-identity-sql.test.ts +++ b/packages/db/src/contractor-identity-sql.test.ts @@ -29,7 +29,28 @@ VALUES '103267195', ''), ('eop:contracts:test', '2026-07-18T00:00:00Z', 'UNP-NULL', 'C-NULL', '2026-07-01', '2026-07-02', '123456786', 'Тестов възложител', 'Липсваща самоличност', 400, 400, 'EUR', - NULL, NULL); + NULL, NULL), + ('eop:contracts:test', '2026-07-18T00:00:00Z', 'UNP-FOLD-1', 'C-FOLD-1', '2026-07-01', + '2026-07-02', '123456786', 'Тестов възложител', 'Кавички и регистър 1', 10, 10, 'EUR', + '', '„Строй Инвест" ЕООД'), + ('eop:contracts:test', '2026-07-18T00:00:00Z', 'UNP-FOLD-2', 'C-FOLD-2', '2026-07-01', + '2026-07-02', '123456786', 'Тестов възложител', 'Кавички и регистър 2', 10, 10, 'EUR', + '', '"СТРОЙ ИНВЕСТ" ЕООД'), + ('eop:contracts:test', '2026-07-18T00:00:00Z', 'UNP-FOLD-3', 'C-FOLD-3', '2026-07-01', + '2026-07-02', '123456786', 'Тестов възложител', 'Кавички и интервали', 10, 10, 'EUR', + '', '«строй инвест» ЕООД'), + ('eop:contracts:test', '2026-07-18T00:00:00Z', 'UNP-DASH-EN', 'C-DASH-EN', '2026-07-01', + '2026-07-02', '123456786', 'Тестов възложител', 'Тире 1', 10, 10, 'EUR', + '', '„Марица–Изток"'), + ('eop:contracts:test', '2026-07-18T00:00:00Z', 'UNP-DASH-ASCII', 'C-DASH-ASCII', '2026-07-01', + '2026-07-02', '123456786', 'Тестов възложител', 'Тире 2', 10, 10, 'EUR', + '', '„Марица-Изток"'), + ('eop:contracts:test', '2026-07-18T00:00:00Z', 'UNP-GUARD-HYPHEN', 'C-GUARD-HYPHEN', '2026-07-01', + '2026-07-02', '123456786', 'Тестов възложител', 'Пази тирето', 10, 10, 'EUR', + '', '„Марица-Изток"'), + ('eop:contracts:test', '2026-07-18T00:00:00Z', 'UNP-GUARD-SPACE', 'C-GUARD-SPACE', '2026-07-01', + '2026-07-02', '123456786', 'Тестов възложител', 'Пази интервала', 10, 10, 'EUR', + '', '„Марица Изток"'); `; function build(path: 'normalize' | 'refresh'): DatabaseSync { @@ -55,7 +76,14 @@ describe.each(['normalize', 'refresh'] as const)('%s contractor identity', (path .all() as { contract_number: string; bidder_id: string }[]; expect(contracts).toEqual([ + { contract_number: 'C-DASH-ASCII', bidder_id: 'name:МАРИЦА-ИЗТОК' }, + { contract_number: 'C-DASH-EN', bidder_id: 'name:МАРИЦА-ИЗТОК' }, { contract_number: 'C-EMPTY', bidder_id: 'unknown:анонимен' }, + { contract_number: 'C-FOLD-1', bidder_id: 'name:СТРОЙ ИНВЕСТ ЕООД' }, + { contract_number: 'C-FOLD-2', bidder_id: 'name:СТРОЙ ИНВЕСТ ЕООД' }, + { contract_number: 'C-FOLD-3', bidder_id: 'name:СТРОЙ ИНВЕСТ ЕООД' }, + { contract_number: 'C-GUARD-HYPHEN', bidder_id: 'name:МАРИЦА-ИЗТОК' }, + { contract_number: 'C-GUARD-SPACE', bidder_id: 'name:МАРИЦА ИЗТОК' }, { contract_number: 'C-NULL', bidder_id: 'unknown:анонимен' }, { contract_number: 'C-TYPO', bidder_id: 'name:ТИПО ООД' }, { contract_number: 'C-VALID', bidder_id: 'eik:103267194' }, @@ -87,7 +115,7 @@ describe.each(['normalize', 'refresh'] as const)('%s contractor identity', (path (SELECT SUM(won_eur) FROM company_totals) AS company_total`, ) .get() as { raw_total: number; contract_total: number; company_total: number }; - expect(sums).toEqual({ raw_total: 1000, contract_total: 1000, company_total: 1000 }); + expect(sums).toEqual({ raw_total: 1070, contract_total: 1070, company_total: 1070 }); expect( db diff --git a/scripts/normalize-raw.sql b/scripts/normalize-raw.sql index c72a5f2a..a6f9136a 100644 --- a/scripts/normalize-raw.sql +++ b/scripts/normalize-raw.sql @@ -235,22 +235,23 @@ DROP TABLE IF EXISTS contractor_identity; CREATE TABLE contractor_identity ( eik_raw TEXT, name_raw TEXT, + name_norm TEXT, eik_clean TEXT, eik_valid INTEGER NOT NULL, bidder_key TEXT NOT NULL, PRIMARY KEY (eik_raw, name_raw) ); -INSERT INTO contractor_identity (eik_raw, name_raw, eik_clean, eik_valid, bidder_key) +INSERT INTO contractor_identity (eik_raw, name_raw, name_norm, eik_clean, eik_valid, bidder_key) SELECT eik_raw, name_raw, + name_raw, eik_clean, eik_valid, CASE WHEN eik_valid = 1 THEN 'eik:' || eik_clean - WHEN name_raw IS NOT NULL AND TRIM(name_raw) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(name_raw, ' ', ' '), ' ', ' '))) - ELSE 'unknown:анонимен' + ELSE 'pending' END FROM ( SELECT @@ -295,6 +296,31 @@ FROM ( ) ); +UPDATE contractor_identity +SET name_norm = TRIM(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(name_norm, char(160), ' '), char(9), ' '), char(10), ' '), char(13), ' '), ' ', ' '), ' ', ' '), ' ', ' ')); + +UPDATE contractor_identity +SET name_norm = REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(name_norm, '"', ''), '''', ''), '`', ''), '„', ''), '“', ''), '”', ''), '‚', ''), '‘', ''), '’', ''), '«', ''), '»', ''), '′', ''), '″', ''); + +UPDATE contractor_identity +SET name_norm = REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(name_norm, '‐', '-'), '‑', '-'), '‒', '-'), '–', '-'), '—', '-'), '―', '-'), '−', '-'), char(173), ''), char(8203), ''); + +UPDATE contractor_identity +SET name_norm = TRIM(REPLACE(REPLACE(REPLACE(name_norm, ' ', ' '), ' ', ' '), ' ', ' ')); + +UPDATE contractor_identity +SET name_norm = REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(name_norm, 'а', 'А'), 'б', 'Б'), 'в', 'В'), 'г', 'Г'), 'д', 'Д'), 'е', 'Е'), 'ж', 'Ж'), 'з', 'З'), 'и', 'И'), 'й', 'Й'), 'к', 'К'), 'л', 'Л'), 'м', 'М'), 'н', 'Н'), 'о', 'О'); + +UPDATE contractor_identity +SET name_norm = UPPER(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(name_norm, 'п', 'П'), 'р', 'Р'), 'с', 'С'), 'т', 'Т'), 'у', 'У'), 'ф', 'Ф'), 'х', 'Х'), 'ц', 'Ц'), 'ч', 'Ч'), 'ш', 'Ш'), 'щ', 'Щ'), 'ъ', 'Ъ'), 'ь', 'Ь'), 'ю', 'Ю'), 'я', 'Я')); + +UPDATE contractor_identity +SET bidder_key = CASE + WHEN eik_valid = 1 THEN bidder_key + WHEN name_raw IS NOT NULL AND TRIM(name_raw) <> '' THEN 'name:' || name_norm + ELSE 'unknown:анонимен' +END; + -- 4a) Bidders — valid ЕИК first, then normalised name, then one labelled unknown bucket. The -- bucket keeps identity-poor contracts attached without polluting bulstat/eik_normalized. INSERT OR IGNORE INTO bidders (id, name, bulstat, eik_normalized, eik_valid, is_consortium, kind) diff --git a/scripts/refresh-slice.sql b/scripts/refresh-slice.sql index 4c383499..c1e074d1 100644 --- a/scripts/refresh-slice.sql +++ b/scripts/refresh-slice.sql @@ -50,22 +50,23 @@ DROP TABLE IF EXISTS contractor_identity; CREATE TABLE contractor_identity ( eik_raw TEXT, name_raw TEXT, + name_norm TEXT, eik_clean TEXT, eik_valid INTEGER NOT NULL, bidder_key TEXT NOT NULL, PRIMARY KEY (eik_raw, name_raw) ); -INSERT INTO contractor_identity (eik_raw, name_raw, eik_clean, eik_valid, bidder_key) +INSERT INTO contractor_identity (eik_raw, name_raw, name_norm, eik_clean, eik_valid, bidder_key) SELECT eik_raw, name_raw, + name_raw, eik_clean, eik_valid, CASE WHEN eik_valid = 1 THEN 'eik:' || eik_clean - WHEN name_raw IS NOT NULL AND TRIM(name_raw) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(name_raw, ' ', ' '), ' ', ' '))) - ELSE 'unknown:анонимен' + ELSE 'pending' END FROM ( SELECT @@ -110,6 +111,31 @@ FROM ( ) ); +UPDATE contractor_identity +SET name_norm = TRIM(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(name_norm, char(160), ' '), char(9), ' '), char(10), ' '), char(13), ' '), ' ', ' '), ' ', ' '), ' ', ' ')); + +UPDATE contractor_identity +SET name_norm = REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(name_norm, '"', ''), '''', ''), '`', ''), '„', ''), '“', ''), '”', ''), '‚', ''), '‘', ''), '’', ''), '«', ''), '»', ''), '′', ''), '″', ''); + +UPDATE contractor_identity +SET name_norm = REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(name_norm, '‐', '-'), '‑', '-'), '‒', '-'), '–', '-'), '—', '-'), '―', '-'), '−', '-'), char(173), ''), char(8203), ''); + +UPDATE contractor_identity +SET name_norm = TRIM(REPLACE(REPLACE(REPLACE(name_norm, ' ', ' '), ' ', ' '), ' ', ' ')); + +UPDATE contractor_identity +SET name_norm = REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(name_norm, 'а', 'А'), 'б', 'Б'), 'в', 'В'), 'г', 'Г'), 'д', 'Д'), 'е', 'Е'), 'ж', 'Ж'), 'з', 'З'), 'и', 'И'), 'й', 'Й'), 'к', 'К'), 'л', 'Л'), 'м', 'М'), 'н', 'Н'), 'о', 'О'); + +UPDATE contractor_identity +SET name_norm = UPPER(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(name_norm, 'п', 'П'), 'р', 'Р'), 'с', 'С'), 'т', 'Т'), 'у', 'У'), 'ф', 'Ф'), 'х', 'Х'), 'ц', 'Ц'), 'ч', 'Ч'), 'ш', 'Ш'), 'щ', 'Щ'), 'ъ', 'Ъ'), 'ь', 'Ь'), 'ю', 'Ю'), 'я', 'Я')); + +UPDATE contractor_identity +SET bidder_key = CASE + WHEN eik_valid = 1 THEN bidder_key + WHEN name_raw IS NOT NULL AND TRIM(name_raw) <> '' THEN 'name:' || name_norm + ELSE 'unknown:анонимен' +END; + INSERT OR IGNORE INTO bidders (id, name, bulstat, eik_normalized, eik_valid, is_consortium, kind) SELECT bidder_key, From c10a89738127d2acceaef7abcc86e5e79a56c4c9 Mon Sep 17 00:00:00 2001 From: Todor Kolev Date: Mon, 20 Jul 2026 14:43:09 +0000 Subject: [PATCH 07/17] =?UTF-8?q?feat(db):=20pick=20joint=20leads=20by=20?= =?UTF-8?q?=D0=A3=D0=9D=D0=9F=20prefix=20and=20value=20the=20participation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three refinements to the joint-procurement attribution: - Lead selection gains a top tier: the УНП prefix. УНП is `--`, so the prefix identifies the authority that REGISTERED the procedure, i.e. the organiser. The map is learned from single-authority rows, where 4030 prefixes resolve to exactly one authority. Measured against the 354 joint tenders in the corpus, the prefix agrees with the previous first-EIK fallback 292 times, corrects it twice, and never points outside the member list (60 have no usable prefix). Name match and first EIK stay as the lower tiers. - `authority_joint_participation` gains `joint_contract_value_eur`: the total value of the joint procurements an authority took part in. It is informational only and is never summed into `authority_totals.spent_eur` or any national/leaderboard total, so a non-lead co-authority still shows zero spend for the contract while its page can state the amount involved. - `contract_co_authorities` and `authority_joint_participation` were defined both in `0000_init.sql` and in `0002_contract_co_authorities.sql`. The ETL work DB is built from 0000 only, so 0000 is the single home and the redundant 0002 is deleted; its table assertions stay in migrations.test.ts, now covering 0000. refresh-slice mirrors all of it. It learns the prefix map from the existing tender history rather than raw staging, which holds only the touched slice. --- packages/db/migrations/0000_init.sql | 8 +- .../0002_contract_co_authorities.sql | 18 --- packages/db/src/migrations.test.ts | 2 - packages/db/src/refresh-slice.test.ts | 135 ++++++++++++++++++ scripts/normalize-raw.sql | 41 +++++- scripts/precompute.sql | 17 ++- scripts/refresh-slice.sql | 46 +++++- 7 files changed, 226 insertions(+), 41 deletions(-) delete mode 100644 packages/db/migrations/0002_contract_co_authorities.sql diff --git a/packages/db/migrations/0000_init.sql b/packages/db/migrations/0000_init.sql index 7af2f46b..eadbe728 100644 --- a/packages/db/migrations/0000_init.sql +++ b/packages/db/migrations/0000_init.sql @@ -255,11 +255,13 @@ CREATE TABLE authority_totals ( last_date TEXT ); --- Non-monetary joint-procurement participation rollup. It is deliberately separate from --- authority_totals so neither spent_eur nor the ordinary lead-contract count can double-count. +-- Joint-procurement participation rollup. joint_contract_value_eur is the total value of the joint +-- procurements the authority took part in; it is informational only and must NEVER be summed into +-- authority_totals.spent_eur or any national/leaderboard total. CREATE TABLE authority_joint_participation ( authority_id TEXT PRIMARY KEY REFERENCES authorities(id), - joint_contract_participations INTEGER NOT NULL + joint_contract_participations INTEGER NOT NULL, + joint_contract_value_eur REAL NOT NULL DEFAULT 0 ); -- Per CPV division. Sector facet + filter counts on the list pages. diff --git a/packages/db/migrations/0002_contract_co_authorities.sql b/packages/db/migrations/0002_contract_co_authorities.sql deleted file mode 100644 index a461e4e9..00000000 --- a/packages/db/migrations/0002_contract_co_authorities.sql +++ /dev/null @@ -1,18 +0,0 @@ --- Joint procurements retain one lead authority on tenders, while this bridge records every --- participating authority. ordinal = 0 is always the lead; remaining members keep source order. -CREATE TABLE IF NOT EXISTS contract_co_authorities ( - contract_id TEXT NOT NULL REFERENCES contracts(id) ON DELETE CASCADE, - authority_id TEXT NOT NULL REFERENCES authorities(id), - ordinal INTEGER NOT NULL CHECK (ordinal >= 0), - PRIMARY KEY (contract_id, authority_id) -); - -CREATE INDEX IF NOT EXISTS idx_contract_co_authorities_authority - ON contract_co_authorities(authority_id, contract_id); - --- Non-monetary participation stays outside authority_totals, whose spend/count reconciliation is --- intentionally unchanged and lead-only. -CREATE TABLE IF NOT EXISTS authority_joint_participation ( - authority_id TEXT PRIMARY KEY REFERENCES authorities(id), - joint_contract_participations INTEGER NOT NULL -); diff --git a/packages/db/src/migrations.test.ts b/packages/db/src/migrations.test.ts index 3815eadd..f0fcb985 100644 --- a/packages/db/src/migrations.test.ts +++ b/packages/db/src/migrations.test.ts @@ -9,7 +9,6 @@ import { describe, expect, it } from 'vitest'; const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); const migration0 = resolve(root, 'packages/db/migrations/0000_init.sql'); const migration1 = resolve(root, 'packages/db/migrations/0001_flow_pairs_bidder_index.sql'); -const migration2 = resolve(root, 'packages/db/migrations/0002_contract_co_authorities.sql'); function sqlite(dbPath: string, sql: string): string { return execFileSync('sqlite3', [dbPath], { input: sql, encoding: 'utf8' }); @@ -28,7 +27,6 @@ describe('served migrations', () => { try { readScript(dbPath, migration0); readScript(dbPath, migration1); - readScript(dbPath, migration2); expect( sqlite( diff --git a/packages/db/src/refresh-slice.test.ts b/packages/db/src/refresh-slice.test.ts index aa20ec76..d90f93cd 100644 --- a/packages/db/src/refresh-slice.test.ts +++ b/packages/db/src/refresh-slice.test.ts @@ -11,6 +11,7 @@ const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); const schemaPath = resolve(root, 'packages/db/migrations/0000_init.sql'); const refreshSlicePath = resolve(root, 'scripts/refresh-slice.sql'); const normalizePath = resolve(root, 'scripts/normalize-raw.sql'); +const precomputePath = resolve(root, 'scripts/precompute.sql'); const workStagingSchemaPath = resolve(root, 'scripts/work-staging-schema.sql'); function sqlite(dbPath: string, sql: string): string { @@ -352,6 +353,140 @@ function seedTwoCollidingBidders(dbPath: string): void { } } +function seedJointAuthorityFixture(dbPath: string, refresh: boolean): void { + if (refresh) { + sqlite( + dbPath, + `INSERT INTO authorities (id, name, bulstat, type) + VALUES ('auth:222222222', 'Second Authority', '222222222', 'public'); + INSERT INTO tenders + (id, source_id, title, authority_id, currency, procedure_type, status) + VALUES + ('t:00080-2023-0001', '00080-2023-0001', 'Historical prefix observation', + 'auth:222222222', 'EUR', 'open', 'published');`, + ); + } + + sqlite( + dbPath, + `INSERT INTO raw_tenders + (source, dataset_year, fetched_at, unp, tender_id, procedure_type, procurement_subject, + estimated_value, currency, authority_name, authority_eik, authority_type, lot_id, published_at) + VALUES + ('eop:tenders:2024-01-01', 2024, '2024-01-02T00:00:00Z', '00080-2023-0001', + 'TENDER-PREFIX-HISTORY', 'open', 'Historical prefix observation', 1000, 'EUR', + 'Second Authority', '222222222', 'public', NULL, '2023-01-01'), + ('eop:tenders:2024-01-01', 2024, '2024-01-02T00:00:00Z', '99999-2023-0001', + 'TENDER-NAME-HISTORY', 'open', 'Historical name observation', 1000, 'EUR', + 'First Authority', '111111111', 'public', NULL, '2023-01-01'), + ('eop:tenders:2024-01-01', 2024, '2024-01-02T00:00:00Z', '00080-2024-0030', + 'TENDER-JOINT-PREFIX', 'open', 'Prefix-led joint tender', 1000, 'EUR', + 'First Authority', '111111111;222222222', 'public', NULL, '2024-01-01'), + ('eop:tenders:2024-01-01', 2024, '2024-01-02T00:00:00Z', 'UNRESOLVED-2024-0001', + 'TENDER-JOINT-FALLBACK', 'open', 'Fallback joint tender', 1000, 'EUR', + 'Third Authority', '333333333;444444444', 'public', NULL, '2024-01-01'); + + INSERT INTO raw_contracts + (source, dataset_year, dataset_variant, fetched_at, needs_enrichment, document_number, + published_at, unp, tender_ext_id, procedure_type, procurement_subject, estimated_value, + procurement_currency, authority_name, authority_eik, authority_type, contract_number, + contract_date, signing_value, currency, contract_subject, awarded_to_group, + contractor_eik, contractor_name, eu_funded, bids_received) + VALUES + ('eop:contracts:2024-01-01', 2024, 'eop', '2024-01-02T00:00:00Z', 0, + 'DOC-JOINT-PREFIX', '2024-01-01', '00080-2024-0030', 'TENDER-JOINT-PREFIX', 'open', + 'Prefix-led joint tender', 1000, 'EUR', 'First Authority', '111111111;222222222', + 'public', 'CONTRACT-JOINT-PREFIX', '2024-01-02', 100, 'EUR', 'Prefix-led contract', 0, + '777777777', 'Joint Bidder', 0, 1), + ('eop:contracts:2024-01-01', 2024, 'eop', '2024-01-02T00:00:00Z', 0, + 'DOC-JOINT-FALLBACK', '2024-01-01', 'UNRESOLVED-2024-0001', + 'TENDER-JOINT-FALLBACK', 'open', 'Fallback joint tender', 1000, 'EUR', + 'Third Authority', '333333333;444444444', 'public', 'CONTRACT-JOINT-FALLBACK', + '2024-01-02', 200, 'EUR', 'Fallback contract', 0, '777777777', 'Joint Bidder', 0, 1);`, + ); +} + +describe('joint-procurement authority attribution', () => { + it.each([ + ['full normalize', false], + ['slice refresh', true], + ] as const)( + 'uses the УНП prefix and keeps participation value non-summable on %s', + (_label, refresh) => { + const dir = mkdtempSync(resolve(tmpdir(), 'sigma-joint-authorities-')); + const dbPath = resolve(dir, 'test.sqlite'); + try { + initWorkDb(dbPath); + seedJointAuthorityFixture(dbPath, refresh); + readScript(dbPath, refresh ? refreshSlicePath : normalizePath); + if (!refresh) readScript(dbPath, precomputePath); + + expect( + sqliteJson<{ source_id: string; authority_id: string }>( + dbPath, + `SELECT source_id, authority_id FROM tenders + WHERE source_id IN ('00080-2024-0030', 'UNRESOLVED-2024-0001') + ORDER BY source_id`, + ), + ).toEqual([ + { source_id: '00080-2024-0030', authority_id: 'auth:222222222' }, + { source_id: 'UNRESOLVED-2024-0001', authority_id: 'auth:333333333' }, + ]); + + expect( + sqliteJson<{ source_id: string; authority_id: string; ordinal: number }>( + dbPath, + `SELECT t.source_id, cca.authority_id, cca.ordinal + FROM contract_co_authorities cca + JOIN contracts c ON c.id = cca.contract_id + JOIN tenders t ON t.id = c.tender_id + WHERE cca.ordinal = 0 + ORDER BY t.source_id`, + ), + ).toEqual([ + { source_id: '00080-2024-0030', authority_id: 'auth:222222222', ordinal: 0 }, + { source_id: 'UNRESOLVED-2024-0001', authority_id: 'auth:333333333', ordinal: 0 }, + ]); + + expect( + sqliteJson<{ + authority_id: string; + joint_contract_participations: number; + joint_contract_value_eur: number; + }>( + dbPath, + `SELECT authority_id, joint_contract_participations, joint_contract_value_eur + FROM authority_joint_participation + WHERE authority_id IN ('auth:111111111', 'auth:222222222') + ORDER BY authority_id`, + ), + ).toEqual([ + { + authority_id: 'auth:111111111', + joint_contract_participations: 1, + joint_contract_value_eur: 100, + }, + { + authority_id: 'auth:222222222', + joint_contract_participations: 1, + joint_contract_value_eur: 100, + }, + ]); + + expect( + sqliteJson<{ spent_eur: number }>( + dbPath, + `SELECT COALESCE((SELECT spent_eur FROM authority_totals + WHERE authority_id = 'auth:111111111'), 0) AS spent_eur`, + )[0]?.spent_eur, + ).toBe(0); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }, + ); +}); + describe('refresh-slice EOP base derivation', () => { it('derives new eop base rows as c:e contracts and is idempotent', () => { const dir = mkdtempSync(resolve(tmpdir(), 'sigma-refresh-slice-')); diff --git a/scripts/normalize-raw.sql b/scripts/normalize-raw.sql index 87ad12bf..dc6b2639 100644 --- a/scripts/normalize-raw.sql +++ b/scripts/normalize-raw.sql @@ -32,6 +32,7 @@ -- Full clear in child→parent order (D1 enforces FKs). DROP TABLE IF EXISTS joint_tender_leads; +DROP TABLE IF EXISTS unp_prefix_authorities; DROP TABLE IF EXISTS joint_authority_members; DROP TABLE IF EXISTS joint_tender_sources; DELETE FROM search_index; @@ -168,9 +169,35 @@ INSERT OR IGNORE INTO authorities (id, name, bulstat, type) SELECT authority_id, authority_name, authority_eik, authority_type FROM member_defaults; --- Lead default: match the tender's modal raw authority name to a co-authority's modal standalone --- name. Joined/unmatched names fall back to the first EIK. This scratch-only mode does not alter --- displayed canonical names (that remains the separate #194 concern). +-- Learn the modal standalone authority for each valid УНП prefix. The prefix identifies the +-- authority that registered the procedure and therefore outranks the name/first-EIK fallbacks. +CREATE TABLE unp_prefix_authorities ( + prefix TEXT PRIMARY KEY, + authority_eik TEXT NOT NULL +); +WITH prefix_observations AS ( + SELECT SUBSTR(unp, 1, 5) AS prefix, authority_eik FROM raw_contracts + WHERE authority_eik IS NOT NULL AND authority_eik NOT LIKE '%;%' + AND unp GLOB '[0-9][0-9][0-9][0-9][0-9]-*' + UNION ALL + SELECT SUBSTR(unp, 1, 5) AS prefix, authority_eik FROM raw_tenders + WHERE authority_eik IS NOT NULL AND authority_eik NOT LIKE '%;%' + AND unp GLOB '[0-9][0-9][0-9][0-9][0-9]-*' +), ranked AS ( + SELECT prefix, authority_eik, + ROW_NUMBER() OVER ( + PARTITION BY prefix + ORDER BY COUNT(*) DESC, authority_eik + ) AS rn + FROM prefix_observations + GROUP BY prefix, authority_eik +) +INSERT INTO unp_prefix_authorities (prefix, authority_eik) +SELECT prefix, authority_eik FROM ranked WHERE rn = 1; + +-- Lead default: prefer the УНП-prefix authority, then match the tender's modal raw authority name +-- to a co-authority's modal standalone name. Unmatched rows fall back to the first EIK. This +-- scratch-only mode does not alter displayed canonical names (that remains the separate #194 concern). CREATE TABLE joint_tender_leads ( unp TEXT PRIMARY KEY, authority_id TEXT NOT NULL REFERENCES authorities(id) @@ -199,12 +226,17 @@ WITH standalone_observations AS ( SELECT m.unp, m.authority_id, ROW_NUMBER() OVER ( PARTITION BY m.unp - ORDER BY CASE WHEN c.authority_name = s.authority_name THEN 0 ELSE 1 END, + ORDER BY CASE WHEN p.authority_eik = SUBSTR(m.authority_id, 6) THEN 0 ELSE 1 END, + CASE WHEN c.authority_name = s.authority_name THEN 0 ELSE 1 END, m.source_ordinal, m.authority_id ) AS rn FROM joint_authority_members m JOIN joint_tender_sources s ON s.unp = m.unp LEFT JOIN canonical_names c ON c.authority_eik = SUBSTR(m.authority_id, 6) + LEFT JOIN unp_prefix_authorities p + ON p.prefix = CASE + WHEN m.unp GLOB '[0-9][0-9][0-9][0-9][0-9]-*' THEN SUBSTR(m.unp, 1, 5) + END ) INSERT INTO joint_tender_leads (unp, authority_id) SELECT unp, authority_id FROM ranked WHERE rn = 1; @@ -998,6 +1030,7 @@ SELECT 1, datetime('now'); DROP TABLE joint_tender_leads; +DROP TABLE unp_prefix_authorities; DROP TABLE joint_authority_members; DROP TABLE joint_tender_sources; diff --git a/scripts/precompute.sql b/scripts/precompute.sql index 68b4c313..f1bced6e 100644 --- a/scripts/precompute.sql +++ b/scripts/precompute.sql @@ -92,17 +92,20 @@ UPDATE authority_totals SET primary_sector = ( WHERE t.authority_id = authority_totals.authority_id AND c.amount_eur IS NOT NULL AND COALESCE(t.cpv_code,'') <> '' GROUP BY substr(t.cpv_code, 1, 2) ORDER BY SUM(c.amount_eur) DESC, substr(t.cpv_code, 1, 2) LIMIT 1); --- Separate, non-monetary participation metric. It counts every joint-contract bridge association, --- including the lead, and never feeds authority_totals.spent_eur or its ordinary contract count. +-- Separate participation metrics. They count every joint-contract bridge association, including +-- the lead; the value is informational only and never feeds authority_totals or any national total. CREATE TABLE IF NOT EXISTS authority_joint_participation ( authority_id TEXT PRIMARY KEY REFERENCES authorities(id), - joint_contract_participations INTEGER NOT NULL + joint_contract_participations INTEGER NOT NULL, + joint_contract_value_eur REAL NOT NULL DEFAULT 0 ); DELETE FROM authority_joint_participation; -INSERT INTO authority_joint_participation (authority_id, joint_contract_participations) -SELECT authority_id, COUNT(*) -FROM contract_co_authorities -GROUP BY authority_id; +INSERT INTO authority_joint_participation + (authority_id, joint_contract_participations, joint_contract_value_eur) +SELECT cca.authority_id, COUNT(*), COALESCE(SUM(c.amount_eur), 0) +FROM contract_co_authorities cca +JOIN contracts c ON c.id = cca.contract_id +GROUP BY cca.authority_id; -- home_totals uses the browsable leaderboard grains for authority/bidder counts, and the same -- freshness definition as refresh-slice.sql: latest in-corpus signed contract date. diff --git a/scripts/refresh-slice.sql b/scripts/refresh-slice.sql index ed703db0..76134d05 100644 --- a/scripts/refresh-slice.sql +++ b/scripts/refresh-slice.sql @@ -17,6 +17,7 @@ DROP TABLE IF EXISTS refresh_touched_contracts; DROP TABLE IF EXISTS refresh_touched_bidders; DROP TABLE IF EXISTS refresh_touched_authorities; DROP TABLE IF EXISTS refresh_joint_tender_leads; +DROP TABLE IF EXISTS refresh_unp_prefix_authorities; DROP TABLE IF EXISTS refresh_joint_authority_members; DROP TABLE IF EXISTS refresh_joint_tender_sources; CREATE TABLE refresh_touched_contracts (id TEXT PRIMARY KEY); @@ -139,6 +140,29 @@ WITH name_counts AS ( INSERT OR IGNORE INTO authorities (id, name, bulstat, type) SELECT authority_id, authority_name, authority_eik, authority_type FROM member_defaults; +-- Learn the modal authority for each valid УНП prefix from the full existing tender history. Raw +-- staging contains only the touched slice here, so it cannot provide a stable corpus-wide map. +CREATE TABLE refresh_unp_prefix_authorities ( + prefix TEXT PRIMARY KEY, + authority_eik TEXT NOT NULL +); +WITH prefix_observations AS ( + SELECT SUBSTR(source_id, 1, 5) AS prefix, SUBSTR(authority_id, 6) AS authority_eik + FROM tenders + WHERE authority_id NOT LIKE '%;%' + AND source_id GLOB '[0-9][0-9][0-9][0-9][0-9]-*' +), ranked AS ( + SELECT prefix, authority_eik, + ROW_NUMBER() OVER ( + PARTITION BY prefix + ORDER BY COUNT(*) DESC, authority_eik + ) AS rn + FROM prefix_observations + GROUP BY prefix, authority_eik +) +INSERT INTO refresh_unp_prefix_authorities (prefix, authority_eik) +SELECT prefix, authority_eik FROM ranked WHERE rn = 1; + CREATE TABLE refresh_joint_tender_leads ( unp TEXT PRIMARY KEY, authority_id TEXT NOT NULL REFERENCES authorities(id) @@ -169,12 +193,17 @@ WITH standalone_observations AS ( SELECT m.unp, m.authority_id, ROW_NUMBER() OVER ( PARTITION BY m.unp - ORDER BY CASE WHEN c.authority_name = s.authority_name THEN 0 ELSE 1 END, + ORDER BY CASE WHEN p.authority_eik = SUBSTR(m.authority_id, 6) THEN 0 ELSE 1 END, + CASE WHEN c.authority_name = s.authority_name THEN 0 ELSE 1 END, m.source_ordinal, m.authority_id ) AS rn FROM refresh_joint_authority_members m JOIN refresh_joint_tender_sources s ON s.unp = m.unp LEFT JOIN canonical_names c ON c.authority_eik = SUBSTR(m.authority_id, 6) + LEFT JOIN refresh_unp_prefix_authorities p + ON p.prefix = CASE + WHEN m.unp GLOB '[0-9][0-9][0-9][0-9][0-9]-*' THEN SUBSTR(m.unp, 1, 5) + END ) INSERT INTO refresh_joint_tender_leads (unp, authority_id) SELECT unp, authority_id FROM ranked WHERE rn = 1; @@ -1469,14 +1498,16 @@ UPDATE authority_totals SET primary_sector = ( GROUP BY substr(t.cpv_code, 1, 2) ORDER BY SUM(c.amount_eur) DESC, substr(t.cpv_code, 1, 2) LIMIT 1) WHERE authority_id IN (SELECT authority_id FROM refresh_touched_authorities); --- Joint participation is a separate non-monetary rollup and never enters lead spend/count totals. +-- Joint participation is separate from lead totals; its value is informational and non-summable. DELETE FROM authority_joint_participation WHERE authority_id IN (SELECT authority_id FROM refresh_touched_authorities); -INSERT INTO authority_joint_participation (authority_id, joint_contract_participations) -SELECT authority_id, COUNT(*) -FROM contract_co_authorities -WHERE authority_id IN (SELECT authority_id FROM refresh_touched_authorities) -GROUP BY authority_id; +INSERT INTO authority_joint_participation + (authority_id, joint_contract_participations, joint_contract_value_eur) +SELECT cca.authority_id, COUNT(*), COALESCE(SUM(c.amount_eur), 0) +FROM contract_co_authorities cca +JOIN contracts c ON c.id = cca.contract_id +WHERE cca.authority_id IN (SELECT authority_id FROM refresh_touched_authorities) +GROUP BY cca.authority_id; -- @refresh-batch flow-pairs DELETE FROM flow_pairs; @@ -1552,6 +1583,7 @@ FROM contracts c GROUP BY CASE WHEN c.eu_funded = 1 THEN '1' ELSE '0' END; -- @refresh-batch cleanup DROP TABLE IF EXISTS refresh_joint_tender_leads; +DROP TABLE IF EXISTS refresh_unp_prefix_authorities; DROP TABLE IF EXISTS refresh_joint_authority_members; DROP TABLE IF EXISTS refresh_joint_tender_sources; DROP TABLE IF EXISTS refresh_touched_contracts; From e71c03cfd4ae6e60dcd1784d56a6fb5cccdcb57a Mon Sep 17 00:00:00 2001 From: Bilko Date: Sun, 19 Jul 2026 08:34:52 -0700 Subject: [PATCH 08/17] fix(db,web): convert current_value_eur from the amendment currency, not signing currency (#245) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit contracts.current_value is populated from the latest amendment's value_after, denominated in THAT amendment's own currency — but every EUR conversion of current_value used contracts.currency (the contract's original signing currency), silently re-converting an already-EUR amendment (e.g. one recorded after ЦАИС ЕОП's 2026 BGN->EUR feed switch) and halving the reported value. Add contracts.current_value_currency to track which currency last set current_value, and route every current_value -> EUR conversion (refresh-slice, normalize-raw, precompute, the contract detail query) through it instead of contracts.currency. signing_value/signing_value_eur are unaffected — signing really is denominated in the contract's own currency. --- .../0002_current_value_currency.sql | 6 ++ packages/db/src/integrity-checks.test.ts | 4 ++ packages/db/src/migrations.test.ts | 9 +++ packages/db/src/queries/details.ts | 5 +- packages/db/src/refresh-slice.test.ts | 67 +++++++++++++++++++ scripts/normalize-raw.sql | 43 +++++++++++- scripts/precompute.sql | 4 +- scripts/refresh-slice.sql | 52 +++++++++++--- 8 files changed, 174 insertions(+), 16 deletions(-) create mode 100644 packages/db/migrations/0002_current_value_currency.sql diff --git a/packages/db/migrations/0002_current_value_currency.sql b/packages/db/migrations/0002_current_value_currency.sql new file mode 100644 index 00000000..03f43420 --- /dev/null +++ b/packages/db/migrations/0002_current_value_currency.sql @@ -0,0 +1,6 @@ +-- Track the currency of whichever amendment last set contracts.current_value. +-- current_value can be denominated in a DIFFERENT currency than the contract's +-- original signing currency (contracts.currency) when the latest amendment was +-- recorded after ЦАИС ЕОП's 2026 BGN->EUR feed switch. EUR conversions of +-- current_value must use this column, not contracts.currency. +ALTER TABLE contracts ADD COLUMN current_value_currency TEXT; diff --git a/packages/db/src/integrity-checks.test.ts b/packages/db/src/integrity-checks.test.ts index fb2d1aa4..c14813ed 100644 --- a/packages/db/src/integrity-checks.test.ts +++ b/packages/db/src/integrity-checks.test.ts @@ -21,6 +21,8 @@ import { const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); const schemaPath = resolve(root, 'packages/db/migrations/0000_init.sql'); +const migration1Path = resolve(root, 'packages/db/migrations/0001_flow_pairs_bidder_index.sql'); +const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); const precomputePath = resolve(root, 'scripts/precompute.sql'); function sqlite(dbPath: string, sql: string): void { @@ -61,6 +63,8 @@ function freshDb(): string { const dir = mkdtempSync(resolve(tmpdir(), 'sigma-integrity-')); const dbPath = resolve(dir, 'test.sqlite'); readScript(dbPath, schemaPath); + readScript(dbPath, migration1Path); + readScript(dbPath, migration2Path); sqlite(dbPath, CLEAN_FIXTURE); return dbPath; } diff --git a/packages/db/src/migrations.test.ts b/packages/db/src/migrations.test.ts index 3e26faba..8aa86bbf 100644 --- a/packages/db/src/migrations.test.ts +++ b/packages/db/src/migrations.test.ts @@ -9,6 +9,7 @@ import { describe, expect, it } from 'vitest'; const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); const migration0 = resolve(root, 'packages/db/migrations/0000_init.sql'); const migration1 = resolve(root, 'packages/db/migrations/0001_flow_pairs_bidder_index.sql'); +const migration2 = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); function sqlite(dbPath: string, sql: string): string { return execFileSync('sqlite3', [dbPath], { input: sql, encoding: 'utf8' }); @@ -27,6 +28,7 @@ describe('served migrations', () => { try { readScript(dbPath, migration0); readScript(dbPath, migration1); + readScript(dbPath, migration2); expect( sqlite( @@ -72,6 +74,13 @@ describe('served migrations', () => { expect( sqlite(dbPath, "SELECT COUNT(*) FROM sqlite_master WHERE name LIKE 'raw_%';").trim(), ).toBe('0'); + + expect( + sqlite( + dbPath, + "SELECT COUNT(*) FROM pragma_table_info('contracts') WHERE name='current_value_currency';", + ).trim(), + ).toBe('1'); } finally { rmSync(dir, { recursive: true, force: true }); } diff --git a/packages/db/src/queries/details.ts b/packages/db/src/queries/details.ts index 8b3604f5..36e49e9e 100644 --- a/packages/db/src/queries/details.ts +++ b/packages/db/src/queries/details.ts @@ -391,6 +391,7 @@ interface ContractDetailRow { subcontractor_name: string | null; subcontract_value: number | null; contract_currency: string; + current_value_currency: string | null; // tender title: string; unp: string; @@ -466,6 +467,7 @@ export async function getContract( c.signing_value_eur, c.current_value_eur, c.value_flag, c.date_flag, c.bids_received, c.bids_rejected, c.bids_sme, c.bids_non_eea, c.subcontractor_eik, c.subcontractor_name, c.subcontract_value, c.currency AS contract_currency, + c.current_value_currency, t.title, t.source_id AS unp, t.procedure_type, t.cpv_code, t.cpv_description, t.num_lots, t.eop_tender_id, t.estimated_value, t.currency AS tender_currency, t.start_date, t.end_date, @@ -533,7 +535,8 @@ export async function getContract( const signingEur = r.signing_value_eur ?? eurFromNative(r.signing_value, r.contract_currency, r.fx_rate); const currentRaw = - r.current_value_eur ?? eurFromNative(r.current_value, r.contract_currency, r.fx_rate); + r.current_value_eur ?? + eurFromNative(r.current_value, r.current_value_currency || r.contract_currency, r.fx_rate); const procedureEstimatedEur = eurFromNative( r.estimated_value, r.tender_currency, diff --git a/packages/db/src/refresh-slice.test.ts b/packages/db/src/refresh-slice.test.ts index aa20ec76..f6bbef8a 100644 --- a/packages/db/src/refresh-slice.test.ts +++ b/packages/db/src/refresh-slice.test.ts @@ -9,6 +9,8 @@ import { assertIntegrity } from '../../../scripts/integrity-checks.mjs'; const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); const schemaPath = resolve(root, 'packages/db/migrations/0000_init.sql'); +const migration1Path = resolve(root, 'packages/db/migrations/0001_flow_pairs_bidder_index.sql'); +const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); const refreshSlicePath = resolve(root, 'scripts/refresh-slice.sql'); const normalizePath = resolve(root, 'scripts/normalize-raw.sql'); const workStagingSchemaPath = resolve(root, 'scripts/work-staging-schema.sql'); @@ -175,6 +177,8 @@ function seedOcdsOnlySharedNumber(dbPath: string): void { function initWorkDb(dbPath: string): void { readScript(dbPath, schemaPath); + readScript(dbPath, migration1Path); + readScript(dbPath, migration2Path); readScript(dbPath, workStagingSchemaPath); } @@ -358,6 +362,8 @@ describe('refresh-slice EOP base derivation', () => { const dbPath = resolve(dir, 'test.sqlite'); try { readScript(dbPath, schemaPath); + readScript(dbPath, migration1Path); + readScript(dbPath, migration2Path); readScript(dbPath, workStagingSchemaPath); seedEopBaseDay(dbPath); @@ -437,6 +443,8 @@ describe('refresh-slice EOP base derivation', () => { const dbPath = resolve(dir, 'test.sqlite'); try { readScript(dbPath, schemaPath); + readScript(dbPath, migration1Path); + readScript(dbPath, migration2Path); readScript(dbPath, workStagingSchemaPath); seedEopOnlySharedNumber(dbPath); readScript(dbPath, refreshSlicePath); @@ -485,6 +493,8 @@ describe('refresh-slice EOP base derivation', () => { const dbPath = resolve(dir, 'test.sqlite'); try { readScript(dbPath, schemaPath); + readScript(dbPath, migration1Path); + readScript(dbPath, migration2Path); readScript(dbPath, workStagingSchemaPath); sqlite( dbPath, @@ -528,11 +538,68 @@ describe('refresh-slice EOP base derivation', () => { } }); + it('converts current_value_eur from the amendment currency, not the contract signing currency (#245)', () => { + const dir = mkdtempSync(resolve(tmpdir(), 'sigma-refresh-slice-')); + const dbPath = resolve(dir, 'test.sqlite'); + try { + readScript(dbPath, schemaPath); + readScript(dbPath, migration1Path); + readScript(dbPath, migration2Path); + readScript(dbPath, workStagingSchemaPath); + sqlite( + dbPath, + `INSERT INTO authorities (id, name, bulstat, type) VALUES ('auth:523456789', 'Authority Eur', '523456789', 'public'); + INSERT INTO bidders (id, name, bulstat, eik_normalized, eik_valid, kind) VALUES ('eik:877777777', 'Bidder Eur', '877777777', '877777777', 1, 'company'); + INSERT INTO tenders (id, source_id, title, authority_id, estimated_value, currency, procedure_type, status) + VALUES ('t:UNP-EUR', 'UNP-EUR', 'Eur tender', 'auth:523456789', 5000, 'BGN', 'open', 'awarded'); + INSERT INTO contracts + (id, tender_id, bidder_id, amount, currency, signed_at, contract_number, signing_value, + current_value, current_value_currency, annex_count, value_flag, amount_eur, signing_value_eur, current_value_eur) + VALUES + ('c:e:eurannex', 't:UNP-EUR', 'eik:877777777', 500, 'BGN', '2025-06-02', + 'CONTRACT-EUR', 1000, 500, 'BGN', 1, 'ok', 500 / 1.95583, 1000 / 1.95583, 500 / 1.95583); + INSERT INTO raw_amendments + (source, dataset_year, dataset_variant, fetched_at, seq_no, document_number, + contract_number, contract_date, published_at, unp, authority_eik, authority_name, + procurement_subject, contract_kind, value_before, value_after, value_delta, + currency, description) + VALUES + ('eop:annexes:2026-06-02', 2026, 'eop', '2026-06-08T00:00:00Z', '2', 'AMD-EUR-2', + 'CONTRACT-EUR', '2026-06-02', '2026-06-03', 'UNP-EUR', '523456789', 'Authority Eur', + 'Eur tender', 'works', 500, 2000, 1500, 'EUR', 'Post-switch EUR annex');`, + ); + + readScript(dbPath, refreshSlicePath); + const row = sqliteJson<{ + value_flag: string; + current_value: number; + current_value_currency: string; + current_value_eur: number; + signing_value_eur: number; + }>( + dbPath, + `SELECT value_flag, current_value, current_value_currency, current_value_eur, signing_value_eur + FROM contracts WHERE id = 'c:e:eurannex'`, + )[0]; + expect(row?.value_flag).toBe('ok'); + expect(row?.current_value).toBe(2000); + expect(row?.current_value_currency).toBe('EUR'); + // The amendment's own EUR value, unconverted — NOT divided by the BGN peg a second time. + expect(row?.current_value_eur).toBeCloseTo(2000, 6); + // signing_value stays denominated in the contract's own (BGN) signing currency — unaffected. + expect(row?.signing_value_eur).toBeCloseTo(1000 / 1.95583, 6); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + it('classifies amendment rollups from the contract-row estimated value', () => { const dir = mkdtempSync(resolve(tmpdir(), 'sigma-refresh-slice-')); const dbPath = resolve(dir, 'test.sqlite'); try { readScript(dbPath, schemaPath); + readScript(dbPath, migration1Path); + readScript(dbPath, migration2Path); readScript(dbPath, workStagingSchemaPath); sqlite( dbPath, diff --git a/scripts/normalize-raw.sql b/scripts/normalize-raw.sql index 17cad66e..6e8ff60d 100644 --- a/scripts/normalize-raw.sql +++ b/scripts/normalize-raw.sql @@ -323,13 +323,44 @@ SET ownership_kind = ( -- the rate, and the EUR value are all auditable without joining fx_rates. INSERT OR IGNORE INTO contracts (id, tender_id, bidder_id, amount, currency, signed_at, - contract_number, signing_value, current_value, annex_count, eu_funded, bids_received, + contract_number, signing_value, current_value, current_value_currency, annex_count, eu_funded, bids_received, contract_kind, awarded_to_group, value_flag, date_flag, amount_eur, fx_converted, fx_rate, lot_id, document_number, published_at, contract_subject, eu_programme, duration_days, winner_size, contractor_country, bids_sme, bids_rejected, bids_non_eea, subcontractor_eik, subcontractor_name, subcontract_value, eauction, framework, accelerated, strategic) +-- amendment_winner: the currency of whichever raw_amendments row supplied contract_number's +-- current_value (derive-amendments.sql's own rollup, mirrored here so the winning currency +-- travels alongside the value it minted — computed ONCE over raw_amendments, not per-row). +WITH amendment_dedup AS ( + SELECT *, + 'am:' || COALESCE(unp, '') || ':' || COALESCE(contract_number, '') || ':' || + COALESCE( + NULLIF(document_number, ''), + NULLIF(correction_number, ''), + NULLIF(seq_no, ''), + 'content:' || COALESCE(published_at, '') || ':' || + COALESCE(CAST(value_before AS TEXT), '') || ':' || + COALESCE(CAST(value_after AS TEXT), '') || ':' || + COALESCE(CAST(value_delta AS TEXT), '') || ':' || + COALESCE(currency, '') || ':' || + COALESCE(description, '') + ) AS natural_key + FROM raw_amendments +), amendment_rn AS ( + SELECT *, + ROW_NUMBER() OVER (PARTITION BY natural_key ORDER BY source DESC, id DESC) AS rn + FROM amendment_dedup +), amendment_winner AS ( + SELECT unp, contract_number, currency, + ROW_NUMBER() OVER ( + PARTITION BY unp, contract_number + ORDER BY published_at DESC, natural_key DESC + ) AS win_rn + FROM amendment_rn + WHERE rn = 1 AND value_after IS NOT NULL +) SELECT CASE WHEN x.source LIKE 'eop:%' THEN 'c:e:' || COALESCE(x.unp, '') || ':' || COALESCE(x.contract_number, '') || ':' || @@ -346,6 +377,7 @@ SELECT x.contract_number, x.signing_value, x.current_value, + x.current_value_currency, COALESCE(x.annex_count, 0), x.eu_funded, x.bids_received, @@ -398,6 +430,10 @@ FROM ( WHEN 'annex_suspect' THEN COALESCE(y.signing_value, y.current_value) ELSE COALESCE(y.current_value, y.signing_value) END AS trusted_native, + -- current_value can be denominated in a DIFFERENT currency than the contract's own (when the + -- winning amendment_winner row recorded one) — precompute.sql's later current_value_eur pass + -- uses this column instead of `currency` so that amendment isn't re-converted a second time. + COALESCE(NULLIF(y.amendment_currency, ''), NULLIF(y.currency, ''), 'BGN') AS current_value_currency, -- ECB rates are published on business days only; carry the latest prior rate forward for -- weekend/holiday signings, never future-dated, and cap the fallback at 10 calendar days. CASE WHEN COALESCE(y.currency, 'BGN') NOT IN ('BGN', 'EUR') @@ -525,9 +561,12 @@ FROM ( LIMIT 1 ) END AS proc_est_eur, - t.estimated_value AS proc_est_native + t.estimated_value AS proc_est_native, + aw.currency AS amendment_currency FROM raw_contracts c LEFT JOIN tenders t ON t.id = 't:' || c.unp + LEFT JOIN amendment_winner aw + ON aw.unp = c.unp AND aw.contract_number = c.contract_number AND aw.win_rn = 1 ) c -- EOP always; an OCDS row only when no EOP row shares its contract_number - EOP wins. -- Key is contract_number (the public-procurement contract document number, common to both feeds), NOT unp: diff --git a/scripts/precompute.sql b/scripts/precompute.sql index d52642d1..d764bc57 100644 --- a/scripts/precompute.sql +++ b/scripts/precompute.sql @@ -34,8 +34,8 @@ UPDATE contracts SET ELSE NULL END, current_value_eur = CASE WHEN value_flag IN ('value_suspect','annex_suspect') OR current_value IS NULL THEN NULL - WHEN COALESCE(currency,'BGN') = 'EUR' THEN current_value - WHEN COALESCE(currency,'BGN') = 'BGN' THEN current_value / 1.95583 + WHEN COALESCE(NULLIF(current_value_currency, ''), NULLIF(currency, ''), 'BGN') = 'EUR' THEN current_value + WHEN COALESCE(NULLIF(current_value_currency, ''), NULLIF(currency, ''), 'BGN') = 'BGN' THEN current_value / 1.95583 WHEN fx_rate IS NOT NULL THEN current_value * fx_rate ELSE NULL END; diff --git a/scripts/refresh-slice.sql b/scripts/refresh-slice.sql index e051a7ed..bbf9a342 100644 --- a/scripts/refresh-slice.sql +++ b/scripts/refresh-slice.sql @@ -516,6 +516,7 @@ WHERE id IN ( ); INSERT OR IGNORE INTO contracts (id, tender_id, bidder_id, amount, currency, signed_at, contract_number, signing_value, current_value, + current_value_currency, annex_count, eu_funded, bids_received, contract_kind, awarded_to_group, value_flag, date_flag, amount_eur, fx_converted, fx_rate, signing_value_eur, current_value_eur, lot_id, document_number, published_at, contract_subject, @@ -534,6 +535,7 @@ SELECT x.contract_number, x.signing_value, x.current_value, + x.current_value_currency, 0, x.eu_funded, x.bids_received, @@ -582,8 +584,8 @@ FROM ( END AS signing_value_eur, CASE WHEN q.value_flag IN ('value_suspect', 'annex_suspect') OR q.current_value IS NULL THEN NULL - WHEN COALESCE(q.currency,'BGN') = 'EUR' THEN q.current_value - WHEN COALESCE(q.currency,'BGN') = 'BGN' THEN q.current_value / 1.95583 + WHEN q.current_value_currency = 'EUR' THEN q.current_value + WHEN q.current_value_currency = 'BGN' THEN q.current_value / 1.95583 ELSE q.current_value * q.fx_rate END AS current_value_eur FROM ( @@ -598,6 +600,10 @@ FROM ( WHEN 'annex_suspect' THEN COALESCE(y.signing_value, y.current_value) ELSE COALESCE(y.current_value, y.signing_value) END AS trusted_native, + -- current_value at insert time comes straight from raw_contracts (this contract's own + -- currency) — amendment-driven current_value_currency overrides only apply later, once + -- the amendments table promotion (below) has run. + COALESCE(NULLIF(y.currency, ''), 'BGN') AS current_value_currency, -- fx: EUR as-is, BGN at the peg, foreign at the signing-date ECB rate (NULL if missing) CASE WHEN COALESCE(y.currency,'BGN') NOT IN ('BGN','EUR') THEN ( @@ -760,6 +766,7 @@ WHERE id IN ( INSERT OR IGNORE INTO contracts (id, tender_id, bidder_id, amount, currency, signed_at, contract_number, signing_value, current_value, + current_value_currency, annex_count, eu_funded, bids_received, contract_kind, awarded_to_group, value_flag, date_flag, amount_eur, fx_converted, fx_rate, signing_value_eur, current_value_eur, lot_id, document_number, published_at, contract_subject, @@ -778,6 +785,7 @@ SELECT x.contract_number, x.signing_value, x.current_value, + x.current_value_currency, COALESCE(x.annex_count, 0), x.eu_funded, x.bids_received, @@ -826,8 +834,8 @@ FROM ( END AS signing_value_eur, CASE WHEN q.value_flag IN ('value_suspect', 'annex_suspect') OR q.current_value IS NULL THEN NULL - WHEN COALESCE(q.currency,'BGN') = 'EUR' THEN q.current_value - WHEN COALESCE(q.currency,'BGN') = 'BGN' THEN q.current_value / 1.95583 + WHEN q.current_value_currency = 'EUR' THEN q.current_value + WHEN q.current_value_currency = 'BGN' THEN q.current_value / 1.95583 ELSE q.current_value * q.fx_rate END AS current_value_eur FROM ( @@ -842,6 +850,10 @@ FROM ( WHEN 'annex_suspect' THEN COALESCE(y.signing_value, y.current_value) ELSE COALESCE(y.current_value, y.signing_value) END AS trusted_native, + -- current_value at insert time comes straight from raw_contracts (this contract's own + -- currency) — amendment-driven current_value_currency overrides only apply later, once + -- the amendments table promotion (below) has run. + COALESCE(NULLIF(y.currency, ''), 'BGN') AS current_value_currency, CASE WHEN COALESCE(y.currency,'BGN') NOT IN ('BGN','EUR') THEN ( SELECT f.eur_per_unit @@ -1063,6 +1075,14 @@ SET AND a.value_after IS NOT NULL ORDER BY a.published_at DESC, a.id DESC LIMIT 1 + ), + current_value_currency = ( + SELECT COALESCE(NULLIF(a.currency, ''), contracts.currency) FROM amendments a + WHERE a.unp = substr(contracts.tender_id, 3) + AND a.contract_number = contracts.contract_number + AND a.value_after IS NOT NULL + ORDER BY a.published_at DESC, a.id DESC + LIMIT 1 ) WHERE (id GLOB 'c:[eo]:*' AND EXISTS ( SELECT 1 FROM raw_contracts rc @@ -1076,12 +1096,22 @@ WHERE (id GLOB 'c:[eo]:*' AND EXISTS ( ); WITH contract_base AS ( - SELECT c.id, c.currency, c.signing_value, c.current_value, c.fx_rate, c.value_flag, + SELECT c.id, c.currency, c.signing_value, c.current_value, c.current_value_currency, c.fx_rate, c.value_flag, te.estimated_value AS proc_est_native, CASE - WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN COALESCE(c.current_value, c.signing_value) - WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'BGN' THEN COALESCE(c.current_value, c.signing_value) / 1.95583 - WHEN c.fx_rate IS NOT NULL THEN COALESCE(c.current_value, c.signing_value) * c.fx_rate + -- current_value (when present) is denominated in current_value_currency (whichever + -- amendment last set it), NOT contracts.currency — that's the contract's original + -- signing currency and only applies to the signing_value fallback. + WHEN c.current_value IS NOT NULL THEN + CASE + WHEN COALESCE(NULLIF(c.current_value_currency, ''), NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN c.current_value + WHEN COALESCE(NULLIF(c.current_value_currency, ''), NULLIF(c.currency, ''), 'BGN') = 'BGN' THEN c.current_value / 1.95583 + WHEN c.fx_rate IS NOT NULL THEN c.current_value * c.fx_rate + ELSE NULL + END + WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN c.signing_value + WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'BGN' THEN c.signing_value / 1.95583 + WHEN c.fx_rate IS NOT NULL THEN c.signing_value * c.fx_rate ELSE NULL END AS eff_eur, CASE @@ -1131,7 +1161,7 @@ WITH contract_base AS ( AND a.contract_number = c.contract_number ) ), base AS ( - SELECT id, currency, signing_value, current_value, fx_rate, proc_est_eur, proc_est_native, + SELECT id, currency, signing_value, current_value, current_value_currency, fx_rate, proc_est_eur, proc_est_native, CASE WHEN c.value_flag <> 'annex_suspect' AND NOT (c.current_value IS NOT NULL AND (c.current_value < 0 OR (c.signing_value > 0 AND c.current_value / c.signing_value >= 100))) @@ -1156,8 +1186,8 @@ WITH contract_base AS ( END AS trusted_native, CASE WHEN new_value_flag IN ('value_suspect', 'annex_suspect') OR current_value IS NULL THEN NULL - WHEN COALESCE(currency, 'BGN') = 'EUR' THEN current_value - WHEN COALESCE(currency, 'BGN') = 'BGN' THEN current_value / 1.95583 + WHEN COALESCE(NULLIF(current_value_currency, ''), NULLIF(currency, ''), 'BGN') = 'EUR' THEN current_value + WHEN COALESCE(NULLIF(current_value_currency, ''), NULLIF(currency, ''), 'BGN') = 'BGN' THEN current_value / 1.95583 WHEN fx_rate IS NOT NULL THEN current_value * fx_rate ELSE NULL END AS new_current_value_eur, From 4238cc8e2763d6248b087d39b5036105b8b0a1cb Mon Sep 17 00:00:00 2001 From: Todor Kolev Date: Mon, 20 Jul 2026 21:28:41 +0000 Subject: [PATCH 09/17] fix(etl): route amount_eur through the amendment currency, not the signing currency (#245) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit current_value is minted from the latest amendment, which can be denominated in a different currency than the contract's signing currency. amount_eur — the column every rollup sums — still converted via contracts.currency, so a BGN contract with a EUR amendment carried roughly half its true value across all aggregates while the detail page showed the corrected current_value_eur, a self-contradicting DB. Pair a trusted_currency column with trusted_native so the conversion always uses the currency of the value actually chosen: the amendment currency when current_value was selected, the contract currency on the signing_value fallback (incl. annex_suspect). Mirror the same routing into eff_eur and add an integrity check asserting amount_eur and current_value_eur agree within a cent for value_flag='ok' rows with a current value — the invariant whose absence let this ship. --- packages/db/src/integrity-checks.test.ts | 22 +++ packages/db/src/refresh-slice.test.ts | 164 ++++++++++++++++++++++- scripts/integrity-checks.d.mts | 1 + scripts/integrity-checks.mjs | 36 ++++- scripts/normalize-raw.sql | 99 ++++++++++++-- scripts/refresh-slice.sql | 146 ++++++++++++++++---- 6 files changed, 423 insertions(+), 45 deletions(-) diff --git a/packages/db/src/integrity-checks.test.ts b/packages/db/src/integrity-checks.test.ts index c14813ed..15b4158e 100644 --- a/packages/db/src/integrity-checks.test.ts +++ b/packages/db/src/integrity-checks.test.ts @@ -11,6 +11,7 @@ import { fileURLToPath } from 'node:url'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { assertIntegrity, + checkCurrentAmountParity, checkDateSanity, checkEikValidity, checkNonEmptyCorpus, @@ -106,6 +107,7 @@ describe('reconciliation gate — clean corpus', () => { for (const nm of [ 'non-empty-corpus', 'rollup-reconciliation', + 'current-amount-parity', 'no-negative-values', 'eik-validity', 'date-sanity', @@ -148,6 +150,26 @@ describe('reconciliation gate — injected violations', () => { expect(result.detail).toMatch(/orphan|unattributed/); }); + it('current-amount-parity catches a detail/rollup EUR disagreement over one cent', () => { + const db = track(freshDb()); + sqlite( + db, + "UPDATE contracts SET current_value = 100000, current_value_eur = 100000.02 WHERE id = 'c:1';", + ); + const result = checkCurrentAmountParity(runner(db)); + expect(result.ok).toBe(false); + expect(result.detail).toMatch(/1 ok contract.*amount_eur != current_value_eur/); + }); + + it('current-amount-parity accepts sub-cent floating-point drift', () => { + const db = track(freshDb()); + sqlite( + db, + "UPDATE contracts SET current_value = 100000, current_value_eur = 100000.009 WHERE id = 'c:1';", + ); + expect(checkCurrentAmountParity(runner(db)).ok).toBe(true); + }); + it('no-negative-values catches a negative ok amount_eur (Sigma derivation bug → hard fail)', () => { const db = track(freshDb()); sqlite(db, "UPDATE contracts SET amount_eur = -100 WHERE id = 'c:1';"); diff --git a/packages/db/src/refresh-slice.test.ts b/packages/db/src/refresh-slice.test.ts index f6bbef8a..e7b9a340 100644 --- a/packages/db/src/refresh-slice.test.ts +++ b/packages/db/src/refresh-slice.test.ts @@ -13,6 +13,9 @@ const migration1Path = resolve(root, 'packages/db/migrations/0001_flow_pairs_bid const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); const refreshSlicePath = resolve(root, 'scripts/refresh-slice.sql'); const normalizePath = resolve(root, 'scripts/normalize-raw.sql'); +const deriveAmendmentsPath = resolve(root, 'scripts/derive-amendments.sql'); +const promoteAmendmentsPath = resolve(root, 'scripts/promote-amendments.sql'); +const precomputePath = resolve(root, 'scripts/precompute.sql'); const workStagingSchemaPath = resolve(root, 'scripts/work-staging-schema.sql'); function sqlite(dbPath: string, sql: string): string { @@ -182,6 +185,77 @@ function initWorkDb(dbPath: string): void { readScript(dbPath, workStagingSchemaPath); } +function seedCrossCurrencyAmendment(dbPath: string): void { + sqlite( + dbPath, + `INSERT INTO raw_tenders + (source, dataset_year, fetched_at, unp, tender_id, procedure_type, procurement_subject, + cpv_code, cpv_description, contract_kind, estimated_value, currency, authority_name, + authority_eik, authority_type, published_at) + VALUES + ('eop:tenders:2025-06-01', 2025, '2026-06-08T00:00:00Z', 'UNP-CROSS-CURRENCY', + 'TENDER-CROSS-CURRENCY', 'open', 'Cross-currency tender', '45000000', 'Construction', + 'works', 300000000, 'EUR', 'Authority Cross Currency', '133456789', 'public', + '2025-06-01'); + + INSERT INTO raw_contracts + (source, dataset_year, dataset_variant, fetched_at, needs_enrichment, document_number, + published_at, unp, tender_ext_id, procedure_type, procurement_subject, cpv_code, + cpv_description, contract_kind, estimated_value, procurement_currency, authority_name, + authority_eik, authority_type, contract_number, contract_date, signing_value, currency, + contract_subject, awarded_to_group, contractor_eik, contractor_name) + VALUES + ('eop:contracts:2025-06-01', 2025, 'eop', '2026-06-08T00:00:00Z', 0, + 'DOC-CROSS-CURRENCY', '2025-06-01', 'UNP-CROSS-CURRENCY', 'TENDER-CROSS-CURRENCY', + 'open', 'Cross-currency tender', '45000000', 'Construction', 'works', 300000000, + 'EUR', 'Authority Cross Currency', '133456789', 'public', 'CONTRACT-CROSS-CURRENCY', + '2025-06-02', 136580250, 'BGN', 'Cross-currency contract', 0, '997654321', + 'Bidder Cross Currency'); + + INSERT INTO raw_amendments + (source, dataset_year, dataset_variant, fetched_at, seq_no, document_number, + contract_number, contract_date, published_at, unp, authority_eik, authority_name, + procurement_subject, contract_kind, value_before, value_after, value_delta, currency, + description) + VALUES + ('eop:annexes:2026-06-01', 2026, 'eop', '2026-06-08T00:00:00Z', '1', + 'AMD-CROSS-CURRENCY', 'CONTRACT-CROSS-CURRENCY', '2025-06-02', '2026-06-03', + 'UNP-CROSS-CURRENCY', '133456789', 'Authority Cross Currency', 'Cross-currency tender', + 'works', 136580250, 104748559.44, -31831690.56, 'EUR', + 'Post-switch EUR amendment');`, + ); +} + +function seedCrossCurrencyFlagBoundary(dbPath: string): void { + sqlite( + dbPath, + `INSERT INTO raw_tenders + (source, dataset_year, fetched_at, unp, tender_id, procedure_type, procurement_subject, + cpv_code, estimated_value, currency, authority_name, authority_eik, authority_type) + VALUES + ('eop:tenders:2025-06-01', 2025, '2026-06-08T00:00:00Z', 'UNP-FX-FLAG', + 'TENDER-FX-FLAG', 'open', 'Currency flag tender', '45000000', 1500, 'EUR', + 'Authority Currency Flag', '143456789', 'public'); + INSERT INTO raw_contracts + (source, dataset_year, dataset_variant, fetched_at, needs_enrichment, unp, tender_ext_id, + procedure_type, procurement_subject, cpv_code, estimated_value, procurement_currency, + authority_name, authority_eik, authority_type, contract_number, contract_date, signing_value, + currency, contract_subject, awarded_to_group, contractor_eik, contractor_name) + VALUES + ('eop:contracts:2025-06-01', 2025, 'eop', '2026-06-08T00:00:00Z', 0, + 'UNP-FX-FLAG', 'TENDER-FX-FLAG', 'open', 'Currency flag tender', '45000000', 1500, + 'EUR', 'Authority Currency Flag', '143456789', 'public', 'CONTRACT-FX-FLAG', + '2025-06-02', 10000, 'BGN', 'Currency flag contract', 0, '987654322', + 'Bidder Currency Flag'); + INSERT INTO raw_amendments + (source, dataset_year, dataset_variant, fetched_at, document_number, contract_number, + published_at, unp, value_before, value_after, value_delta, currency) + VALUES + ('eop:annexes:2026-06-01', 2026, 'eop', '2026-06-08T00:00:00Z', 'AMD-FX-FLAG', + 'CONTRACT-FX-FLAG', '2026-06-03', 'UNP-FX-FLAG', 10000, 20000, 10000, 'EUR');`, + ); +} + function seedContractIdFixture(dbPath: string): void { sqlite( dbPath, @@ -574,18 +648,26 @@ describe('refresh-slice EOP base derivation', () => { value_flag: string; current_value: number; current_value_currency: string; + amount_eur: number; current_value_eur: number; signing_value_eur: number; }>( dbPath, - `SELECT value_flag, current_value, current_value_currency, current_value_eur, signing_value_eur + `SELECT value_flag, current_value, current_value_currency, amount_eur, current_value_eur, signing_value_eur FROM contracts WHERE id = 'c:e:eurannex'`, )[0]; expect(row?.value_flag).toBe('ok'); expect(row?.current_value).toBe(2000); expect(row?.current_value_currency).toBe('EUR'); // The amendment's own EUR value, unconverted — NOT divided by the BGN peg a second time. + expect(row?.amount_eur).toBeCloseTo(2000, 6); expect(row?.current_value_eur).toBeCloseTo(2000, 6); + expect( + sqliteJson<{ won_eur: number }>( + dbPath, + "SELECT won_eur FROM company_totals WHERE bidder_id = 'eik:877777777'", + )[0]?.won_eur, + ).toBeCloseTo(2000, 6); // signing_value stays denominated in the contract's own (BGN) signing currency — unaffected. expect(row?.signing_value_eur).toBeCloseTo(1000 / 1.95583, 6); } finally { @@ -593,6 +675,79 @@ describe('refresh-slice EOP base derivation', () => { } }); + it('normalizes amount_eur and full rollups from a non-BGN amendment currency', () => { + const dir = mkdtempSync(resolve(tmpdir(), 'sigma-normalize-currency-')); + const dbPath = resolve(dir, 'test.sqlite'); + try { + initWorkDb(dbPath); + seedCrossCurrencyAmendment(dbPath); + readScript(dbPath, deriveAmendmentsPath); + readScript(dbPath, normalizePath); + readScript(dbPath, promoteAmendmentsPath); + readScript(dbPath, precomputePath); + + const expected = 104748559.44; + const contract = sqliteJson<{ + value_flag: string; + current_value_currency: string; + amount_eur: number; + current_value_eur: number; + }>( + dbPath, + `SELECT value_flag, current_value_currency, amount_eur, current_value_eur + FROM contracts WHERE contract_number = 'CONTRACT-CROSS-CURRENCY'`, + )[0]; + expect(contract?.value_flag).toBe('ok'); + expect(contract?.current_value_currency).toBe('EUR'); + expect(contract?.amount_eur).toBeCloseTo(expected, 2); + expect(contract?.current_value_eur).toBeCloseTo(expected, 2); + + for (const [table, column] of [ + ['company_totals', 'won_eur'], + ['authority_totals', 'spent_eur'], + ['sector_totals', 'value_eur'], + ['home_totals', 'value_eur'], + ['flow_pairs', 'won_eur'], + ] as const) { + expect( + sqliteJson<{ total: number }>(dbPath, `SELECT ${column} AS total FROM ${table}`)[0] + ?.total, + `${table}.${column}`, + ).toBeCloseTo(expected, 2); + } + expect( + sqliteJson<{ amount: number }>( + dbPath, + "SELECT amount FROM search_index WHERE kind = 'contract'", + )[0]?.amount, + ).toBeCloseTo(expected, 2); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('classifies cross-currency amendments identically in full and slice derives', () => { + const dir = mkdtempSync(resolve(tmpdir(), 'sigma-currency-flag-parity-')); + const fullDb = resolve(dir, 'full.sqlite'); + const sliceDb = resolve(dir, 'slice.sqlite'); + try { + for (const dbPath of [fullDb, sliceDb]) { + initWorkDb(dbPath); + seedCrossCurrencyFlagBoundary(dbPath); + readScript(dbPath, deriveAmendmentsPath); + } + readScript(fullDb, normalizePath); + readScript(sliceDb, refreshSlicePath); + + const valueSql = `SELECT value_flag, ROUND(amount_eur, 2) AS amount_eur + FROM contracts WHERE contract_number = 'CONTRACT-FX-FLAG'`; + expect(sqliteJson(fullDb, valueSql)).toEqual([{ value_flag: 'review', amount_eur: 20000 }]); + expect(sqliteJson(sliceDb, valueSql)).toEqual(sqliteJson(fullDb, valueSql)); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + it('classifies amendment rollups from the contract-row estimated value', () => { const dir = mkdtempSync(resolve(tmpdir(), 'sigma-refresh-slice-')); const dbPath = resolve(dir, 'test.sqlite'); @@ -637,7 +792,7 @@ describe('refresh-slice EOP base derivation', () => { VALUES ('eop:annexes:2026-06-01', 2026, 'eop', '2026-06-07T00:00:00Z', '1', 'AMD-MISMATCH', 'CONTRACT-MISMATCH', '2026-06-02', '2026-06-03', 'UNP-MISMATCH', '423456789', 'Authority Mismatch', - 'Mismatch tender', 'works', 1000, 500000, 499000, 'BGN', 'Huge increase');`, + 'Mismatch tender', 'works', 1000, 500000, 499000, 'EUR', 'Huge increase');`, ); readScript(dbPath, refreshSlicePath); @@ -645,13 +800,16 @@ describe('refresh-slice EOP base derivation', () => { value_flag: string; amount: number; amount_eur: number | null; + current_value_currency: string; signing_value_eur: number | null; }>( dbPath, - "SELECT value_flag, amount, amount_eur, signing_value_eur FROM contracts WHERE contract_number = 'CONTRACT-MISMATCH'", + "SELECT value_flag, amount, amount_eur, current_value_currency, signing_value_eur FROM contracts WHERE contract_number = 'CONTRACT-MISMATCH'", )[0]; expect(row?.value_flag).toBe('annex_suspect'); expect(row?.amount).toBe(1000); + expect(row?.current_value_currency).toBe('EUR'); + // The suspect EUR amendment is rejected, so the selected signing_value still converts as BGN. expect(row?.amount_eur).toBeCloseTo(1000 / 1.95583, 6); expect(row?.signing_value_eur).toBeCloseTo(1000 / 1.95583, 6); } finally { diff --git a/scripts/integrity-checks.d.mts b/scripts/integrity-checks.d.mts index 6b2bd0f2..0b4df4ce 100644 --- a/scripts/integrity-checks.d.mts +++ b/scripts/integrity-checks.d.mts @@ -22,6 +22,7 @@ export interface IntegrityResult { export function checkNonEmptyCorpus(runner: IntegrityRunner): IntegrityResult; export function checkRollupReconciliation(runner: IntegrityRunner): IntegrityResult; +export function checkCurrentAmountParity(runner: IntegrityRunner): IntegrityResult; export function checkNoNegativeValues(runner: IntegrityRunner): IntegrityResult; export function checkEikValidity(runner: IntegrityRunner): IntegrityResult; export function checkDateSanity(runner: IntegrityRunner): IntegrityResult; diff --git a/scripts/integrity-checks.mjs b/scripts/integrity-checks.mjs index 56ab0e5a..c3b1f598 100644 --- a/scripts/integrity-checks.mjs +++ b/scripts/integrity-checks.mjs @@ -146,7 +146,34 @@ export function checkRollupReconciliation(runner) { }; } -// 2) No negative values feeding the totals. Two classes, split by who controls the defect: +// 2) A clean amended contract has one canonical EUR value. The detail-page timeline reads +// current_value_eur while every aggregate and search result reads amount_eur, so allowing these two +// columns to disagree makes the served database contradict itself even when each rollup reconciles +// perfectly with amount_eur. One cent is the user-visible precision boundary. +export function checkCurrentAmountParity(runner) { + const name = 'current-amount-parity'; + if (!tableExists(runner, 'contracts')) + return { name, ok: true, skipped: true, detail: 'contracts table absent' }; + const mismatches = num( + scalar( + runner, + "SELECT COUNT(*) AS n FROM contracts WHERE value_flag = 'ok' AND current_value IS NOT NULL " + + 'AND (amount_eur IS NULL OR current_value_eur IS NULL OR ABS(amount_eur - current_value_eur) > 0.01)', + 'n', + ), + ); + return { + name, + ok: mismatches === 0, + skipped: false, + detail: + mismatches === 0 + ? 'all ok current values agree with canonical amount_eur within €0.01' + : `${mismatches} ok contract(s) have amount_eur != current_value_eur by more than €0.01`, + }; +} + +// 3) No negative values feeding the totals. Two classes, split by who controls the defect: // - value_flag='ok' AND amount_eur<0 → HARD fail. A clean row cannot be negative except via a Sigma // derivation bug (e.g. a sign flip); Sigma owns and can fix it. // - any other flag with amount_eur<0 → WARN. normalize keeps value_low rows (set on @@ -204,7 +231,7 @@ export function checkNoNegativeValues(runner) { }; } -// 3) EIK validity (canonical home: bidders). eik_valid=1 ⇒ eik_normalized is a numeric 9/13-digit +// 4) EIK validity (canonical home: bidders). eik_valid=1 ⇒ eik_normalized is a numeric 9/13-digit // ЕИК; eik_valid<>1 ⇒ eik_normalized IS NULL. normalize-raw guarantees this (it sets // eik_normalized only when eik_valid=1); the gate proves the guarantee held. export function checkEikValidity(runner) { @@ -241,7 +268,7 @@ export function checkEikValidity(runner) { }; } -// 4) Date sanity — REPORTED, NOT GATED. Unlike the other checks, signed_at is a pass-through of the +// 5) Date sanity — REPORTED, NOT GATED. Unlike the other checks, signed_at is a pass-through of the // upstream EOP value, not something Sigma derives. An out-of-range date is an upstream record-level // defect (#19–27) that Sigma consumes and cannot correct, so it must NEVER break the daily import — // a single source typo (real example: signed_at='2029-05-14' in the 2024 feed) would otherwise fail @@ -271,7 +298,7 @@ export function checkDateSanity(runner) { }; } -// 5) Staging → domain reconciliation. normalize-raw records, in one row of pipeline_stats, the +// 6) Staging → domain reconciliation. normalize-raw records, in one row of pipeline_stats, the // eligible-candidate count (the SAME expression the summary prints) and the resulting contracts // count. The gate asserts no contract appeared without an eligible candidate (inserted ≤ // candidates — corroborates the orphan check from the staging side) and that a non-empty corpus @@ -323,6 +350,7 @@ export function checkStagingReconciliation(runner) { export const CHECKS = [ checkNonEmptyCorpus, checkRollupReconciliation, + checkCurrentAmountParity, checkNoNegativeValues, checkEikValidity, checkDateSanity, diff --git a/scripts/normalize-raw.sql b/scripts/normalize-raw.sql index 6e8ff60d..a50a0cdc 100644 --- a/scripts/normalize-raw.sql +++ b/scripts/normalize-raw.sql @@ -388,8 +388,8 @@ SELECT CASE WHEN x.value_flag = 'value_suspect' THEN x.proc_est_eur WHEN x.trusted_native IS NULL THEN NULL - WHEN COALESCE(x.currency, 'BGN') = 'EUR' THEN x.trusted_native - WHEN COALESCE(x.currency, 'BGN') = 'BGN' THEN x.trusted_native / 1.95583 + WHEN x.trusted_currency = 'EUR' THEN x.trusted_native + WHEN x.trusted_currency = 'BGN' THEN x.trusted_native / 1.95583 ELSE x.trusted_native * x.fx_rate END, CASE WHEN COALESCE(x.currency, 'BGN') NOT IN ('BGN', 'EUR') THEN 1 ELSE 0 END, @@ -430,10 +430,27 @@ FROM ( WHEN 'annex_suspect' THEN COALESCE(y.signing_value, y.current_value) ELSE COALESCE(y.current_value, y.signing_value) END AS trusted_native, + -- Keep the companion currency paired with the exact native value chosen above. In particular, + -- annex_suspect normally falls back to signing_value, which remains in contracts.currency even + -- when the rejected amendment used a different currency. + CASE y.value_flag + WHEN 'value_suspect' THEN NULL + WHEN 'annex_suspect' THEN CASE + WHEN y.signing_value IS NOT NULL THEN COALESCE(NULLIF(y.currency, ''), 'BGN') + ELSE COALESCE(NULLIF(y.amendment_currency, ''), NULLIF(y.currency, ''), 'BGN') + END + ELSE CASE + WHEN y.current_value IS NOT NULL THEN COALESCE(NULLIF(y.amendment_currency, ''), NULLIF(y.currency, ''), 'BGN') + ELSE COALESCE(NULLIF(y.currency, ''), 'BGN') + END + END AS trusted_currency, -- current_value can be denominated in a DIFFERENT currency than the contract's own (when the -- winning amendment_winner row recorded one) — precompute.sql's later current_value_eur pass -- uses this column instead of `currency` so that amendment isn't re-converted a second time. - COALESCE(NULLIF(y.amendment_currency, ''), NULLIF(y.currency, ''), 'BGN') AS current_value_currency, + CASE WHEN y.current_value IS NOT NULL + THEN COALESCE(NULLIF(y.amendment_currency, ''), NULLIF(y.currency, ''), 'BGN') + ELSE COALESCE(NULLIF(y.currency, ''), 'BGN') + END AS current_value_currency, -- ECB rates are published on business days only; carry the latest prior rate forward for -- weekend/holiday signings, never future-dated, and cap the fallback at 10 calendar days. CASE WHEN COALESCE(y.currency, 'BGN') NOT IN ('BGN', 'EUR') @@ -534,10 +551,22 @@ FROM ( END AS bidder_key FROM ( SELECT c.*, - CASE - WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN COALESCE(c.current_value, c.signing_value) - WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'BGN' THEN COALESCE(c.current_value, c.signing_value) / 1.95583 - ELSE COALESCE(c.current_value, c.signing_value) * ( + CASE WHEN c.current_value IS NOT NULL THEN CASE + WHEN COALESCE(NULLIF(aw.currency, ''), NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN c.current_value + WHEN COALESCE(NULLIF(aw.currency, ''), NULLIF(c.currency, ''), 'BGN') = 'BGN' THEN c.current_value / 1.95583 + ELSE c.current_value * ( + SELECT f.eur_per_unit + FROM fx_rates f + WHERE f.base_currency = COALESCE(NULLIF(aw.currency, ''), NULLIF(c.currency, '')) + AND f.rate_date <= c.contract_date + AND f.rate_date >= date(c.contract_date, '-10 days') + ORDER BY f.rate_date DESC + LIMIT 1 + ) + END ELSE CASE + WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN c.signing_value + WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'BGN' THEN c.signing_value / 1.95583 + ELSE c.signing_value * ( SELECT f.eur_per_unit FROM fx_rates f WHERE f.base_currency = NULLIF(c.currency, '') @@ -546,7 +575,7 @@ FROM ( ORDER BY f.rate_date DESC LIMIT 1 ) - END AS eff_eur, + END END AS eff_eur, CASE WHEN t.estimated_value IS NULL THEN NULL WHEN COALESCE(NULLIF(t.currency, ''), 'BGN') = 'EUR' THEN t.estimated_value @@ -739,6 +768,33 @@ CREATE TABLE IF NOT EXISTS pipeline_stats ( computed_at TEXT NOT NULL ); DELETE FROM pipeline_stats; +WITH amendment_dedup AS ( + SELECT *, + 'am:' || COALESCE(unp, '') || ':' || COALESCE(contract_number, '') || ':' || + COALESCE( + NULLIF(document_number, ''), + NULLIF(correction_number, ''), + NULLIF(seq_no, ''), + 'content:' || COALESCE(published_at, '') || ':' || + COALESCE(CAST(value_before AS TEXT), '') || ':' || + COALESCE(CAST(value_after AS TEXT), '') || ':' || + COALESCE(CAST(value_delta AS TEXT), '') || ':' || + COALESCE(currency, '') || ':' || + COALESCE(description, '') + ) AS natural_key + FROM raw_amendments +), amendment_rn AS ( + SELECT *, ROW_NUMBER() OVER (PARTITION BY natural_key ORDER BY source DESC, id DESC) AS rn + FROM amendment_dedup +), amendment_winner AS ( + SELECT unp, contract_number, currency, + ROW_NUMBER() OVER ( + PARTITION BY unp, contract_number + ORDER BY published_at DESC, natural_key DESC + ) AS win_rn + FROM amendment_rn + WHERE rn = 1 AND value_after IS NOT NULL +) INSERT INTO pipeline_stats (id, contract_candidates, contracts_inserted, computed_at) SELECT 1, (SELECT COUNT(*) FROM ( @@ -805,10 +861,22 @@ SELECT 1, END AS bidder_key FROM ( SELECT c.*, - CASE - WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN COALESCE(c.current_value, c.signing_value) - WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'BGN' THEN COALESCE(c.current_value, c.signing_value) / 1.95583 - ELSE COALESCE(c.current_value, c.signing_value) * ( + CASE WHEN c.current_value IS NOT NULL THEN CASE + WHEN COALESCE(NULLIF(aw.currency, ''), NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN c.current_value + WHEN COALESCE(NULLIF(aw.currency, ''), NULLIF(c.currency, ''), 'BGN') = 'BGN' THEN c.current_value / 1.95583 + ELSE c.current_value * ( + SELECT f.eur_per_unit + FROM fx_rates f + WHERE f.base_currency = COALESCE(NULLIF(aw.currency, ''), NULLIF(c.currency, '')) + AND f.rate_date <= c.contract_date + AND f.rate_date >= date(c.contract_date, '-10 days') + ORDER BY f.rate_date DESC + LIMIT 1 + ) + END ELSE CASE + WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN c.signing_value + WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'BGN' THEN c.signing_value / 1.95583 + ELSE c.signing_value * ( SELECT f.eur_per_unit FROM fx_rates f WHERE f.base_currency = NULLIF(c.currency, '') @@ -817,7 +885,7 @@ SELECT 1, ORDER BY f.rate_date DESC LIMIT 1 ) - END AS eff_eur, + END END AS eff_eur, CASE WHEN t.estimated_value IS NULL THEN NULL WHEN COALESCE(NULLIF(t.currency, ''), 'BGN') = 'EUR' THEN t.estimated_value @@ -832,9 +900,12 @@ SELECT 1, LIMIT 1 ) END AS proc_est_eur, - t.estimated_value AS proc_est_native + t.estimated_value AS proc_est_native, + aw.currency AS amendment_currency FROM raw_contracts c LEFT JOIN tenders t ON t.id = 't:' || c.unp + LEFT JOIN amendment_winner aw + ON aw.unp = c.unp AND aw.contract_number = c.contract_number AND aw.win_rn = 1 ) c -- Eligibility must mirror the INSERT INTO contracts WHERE exactly, so this candidate count is a -- true superset of what lands (inserted <= candidates holds by construction; the gap is only the diff --git a/scripts/refresh-slice.sql b/scripts/refresh-slice.sql index bbf9a342..63e6df42 100644 --- a/scripts/refresh-slice.sql +++ b/scripts/refresh-slice.sql @@ -16,9 +16,37 @@ CREATE INDEX IF NOT EXISTS idx_contracts_tender_id ON contracts(tender_id); DROP TABLE IF EXISTS refresh_touched_contracts; DROP TABLE IF EXISTS refresh_touched_bidders; DROP TABLE IF EXISTS refresh_touched_authorities; +DROP TABLE IF EXISTS refresh_amendment_winners; CREATE TABLE refresh_touched_contracts (id TEXT PRIMARY KEY); CREATE TABLE refresh_touched_bidders (bidder_id TEXT PRIMARY KEY); CREATE TABLE refresh_touched_authorities (authority_id TEXT PRIMARY KEY); +CREATE TABLE refresh_amendment_winners AS +WITH keyed AS ( + SELECT *, + 'am:' || COALESCE(unp, '') || ':' || COALESCE(contract_number, '') || ':' || + COALESCE( + NULLIF(document_number, ''), NULLIF(correction_number, ''), NULLIF(seq_no, ''), + 'content:' || COALESCE(published_at, '') || ':' || + COALESCE(CAST(value_before AS TEXT), '') || ':' || + COALESCE(CAST(value_after AS TEXT), '') || ':' || + COALESCE(CAST(value_delta AS TEXT), '') || ':' || + COALESCE(currency, '') || ':' || COALESCE(description, '') + ) AS natural_key + FROM raw_amendments +), dedup AS ( + SELECT *, ROW_NUMBER() OVER (PARTITION BY natural_key ORDER BY source DESC, id DESC) AS rn + FROM keyed +), winners AS ( + SELECT unp, contract_number, currency, + ROW_NUMBER() OVER ( + PARTITION BY unp, contract_number ORDER BY published_at DESC, natural_key DESC + ) AS win_rn + FROM dedup + WHERE rn = 1 AND value_after IS NOT NULL +) +SELECT unp, contract_number, currency FROM winners WHERE win_rn = 1; +CREATE INDEX idx_refresh_amendment_winners + ON refresh_amendment_winners(unp, contract_number); -- @refresh-batch authorities-bidders -- ── 1) Authorities referenced by OCDS staging (new ones only; INSERT OR IGNORE) ──────────────────── @@ -572,8 +600,8 @@ FROM ( -- so it counts in every sum; it is merely labelled in the UI. annex_suspect uses trusted_native's signing fallback. CASE WHEN q.value_flag = 'value_suspect' THEN q.proc_est_eur - WHEN COALESCE(q.currency,'BGN') = 'EUR' THEN q.trusted_native - WHEN COALESCE(q.currency,'BGN') = 'BGN' THEN q.trusted_native / 1.95583 + WHEN q.trusted_currency = 'EUR' THEN q.trusted_native + WHEN q.trusted_currency = 'BGN' THEN q.trusted_native / 1.95583 ELSE q.trusted_native * q.fx_rate END AS amount_eur, CASE @@ -600,10 +628,24 @@ FROM ( WHEN 'annex_suspect' THEN COALESCE(y.signing_value, y.current_value) ELSE COALESCE(y.current_value, y.signing_value) END AS trusted_native, - -- current_value at insert time comes straight from raw_contracts (this contract's own - -- currency) — amendment-driven current_value_currency overrides only apply later, once - -- the amendments table promotion (below) has run. - COALESCE(NULLIF(y.currency, ''), 'BGN') AS current_value_currency, + CASE y.value_flag + WHEN 'value_suspect' THEN NULL + WHEN 'annex_suspect' THEN CASE + WHEN y.signing_value IS NOT NULL THEN COALESCE(NULLIF(y.currency, ''), 'BGN') + ELSE COALESCE((SELECT NULLIF(w.currency, '') FROM refresh_amendment_winners w + WHERE w.unp = y.unp AND w.contract_number = y.contract_number), NULLIF(y.currency, ''), 'BGN') + END + ELSE CASE + WHEN y.current_value IS NOT NULL THEN COALESCE((SELECT NULLIF(w.currency, '') FROM refresh_amendment_winners w + WHERE w.unp = y.unp AND w.contract_number = y.contract_number), NULLIF(y.currency, ''), 'BGN') + ELSE COALESCE(NULLIF(y.currency, ''), 'BGN') + END + END AS trusted_currency, + CASE WHEN y.current_value IS NOT NULL + THEN COALESCE((SELECT NULLIF(w.currency, '') FROM refresh_amendment_winners w + WHERE w.unp = y.unp AND w.contract_number = y.contract_number), NULLIF(y.currency, ''), 'BGN') + ELSE COALESCE(NULLIF(y.currency, ''), 'BGN') + END AS current_value_currency, -- fx: EUR as-is, BGN at the peg, foreign at the signing-date ECB rate (NULL if missing) CASE WHEN COALESCE(y.currency,'BGN') NOT IN ('BGN','EUR') THEN ( @@ -694,10 +736,25 @@ FROM ( END AS bidder_key FROM ( SELECT c.*, - CASE - WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN COALESCE(c.current_value, c.signing_value) - WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'BGN' THEN COALESCE(c.current_value, c.signing_value) / 1.95583 - ELSE COALESCE(c.current_value, c.signing_value) * ( + CASE WHEN c.current_value IS NOT NULL THEN CASE + WHEN COALESCE((SELECT NULLIF(w.currency, '') FROM refresh_amendment_winners w + WHERE w.unp = c.unp AND w.contract_number = c.contract_number), NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN c.current_value + WHEN COALESCE((SELECT NULLIF(w.currency, '') FROM refresh_amendment_winners w + WHERE w.unp = c.unp AND w.contract_number = c.contract_number), NULLIF(c.currency, ''), 'BGN') = 'BGN' THEN c.current_value / 1.95583 + ELSE c.current_value * ( + SELECT f.eur_per_unit + FROM fx_rates f + WHERE f.base_currency = COALESCE((SELECT NULLIF(w.currency, '') FROM refresh_amendment_winners w + WHERE w.unp = c.unp AND w.contract_number = c.contract_number), NULLIF(c.currency, '')) + AND f.rate_date <= c.contract_date + AND f.rate_date >= date(c.contract_date, '-10 days') + ORDER BY f.rate_date DESC + LIMIT 1 + ) + END ELSE CASE + WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN c.signing_value + WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'BGN' THEN c.signing_value / 1.95583 + ELSE c.signing_value * ( SELECT f.eur_per_unit FROM fx_rates f WHERE f.base_currency = NULLIF(c.currency, '') @@ -706,7 +763,7 @@ FROM ( ORDER BY f.rate_date DESC LIMIT 1 ) - END AS eff_eur, + END END AS eff_eur, CASE WHEN t.estimated_value IS NULL THEN NULL WHEN COALESCE(NULLIF(t.currency, ''), 'BGN') = 'EUR' THEN t.estimated_value @@ -822,8 +879,8 @@ FROM ( -- so it counts in every sum; it is merely labelled in the UI. annex_suspect uses trusted_native's signing fallback. CASE WHEN q.value_flag = 'value_suspect' THEN q.proc_est_eur - WHEN COALESCE(q.currency,'BGN') = 'EUR' THEN q.trusted_native - WHEN COALESCE(q.currency,'BGN') = 'BGN' THEN q.trusted_native / 1.95583 + WHEN q.trusted_currency = 'EUR' THEN q.trusted_native + WHEN q.trusted_currency = 'BGN' THEN q.trusted_native / 1.95583 ELSE q.trusted_native * q.fx_rate END AS amount_eur, CASE @@ -850,10 +907,24 @@ FROM ( WHEN 'annex_suspect' THEN COALESCE(y.signing_value, y.current_value) ELSE COALESCE(y.current_value, y.signing_value) END AS trusted_native, - -- current_value at insert time comes straight from raw_contracts (this contract's own - -- currency) — amendment-driven current_value_currency overrides only apply later, once - -- the amendments table promotion (below) has run. - COALESCE(NULLIF(y.currency, ''), 'BGN') AS current_value_currency, + CASE y.value_flag + WHEN 'value_suspect' THEN NULL + WHEN 'annex_suspect' THEN CASE + WHEN y.signing_value IS NOT NULL THEN COALESCE(NULLIF(y.currency, ''), 'BGN') + ELSE COALESCE((SELECT NULLIF(w.currency, '') FROM refresh_amendment_winners w + WHERE w.unp = y.unp AND w.contract_number = y.contract_number), NULLIF(y.currency, ''), 'BGN') + END + ELSE CASE + WHEN y.current_value IS NOT NULL THEN COALESCE((SELECT NULLIF(w.currency, '') FROM refresh_amendment_winners w + WHERE w.unp = y.unp AND w.contract_number = y.contract_number), NULLIF(y.currency, ''), 'BGN') + ELSE COALESCE(NULLIF(y.currency, ''), 'BGN') + END + END AS trusted_currency, + CASE WHEN y.current_value IS NOT NULL + THEN COALESCE((SELECT NULLIF(w.currency, '') FROM refresh_amendment_winners w + WHERE w.unp = y.unp AND w.contract_number = y.contract_number), NULLIF(y.currency, ''), 'BGN') + ELSE COALESCE(NULLIF(y.currency, ''), 'BGN') + END AS current_value_currency, CASE WHEN COALESCE(y.currency,'BGN') NOT IN ('BGN','EUR') THEN ( SELECT f.eur_per_unit @@ -948,10 +1019,25 @@ FROM ( END AS bidder_key FROM ( SELECT c.*, - CASE - WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN COALESCE(c.current_value, c.signing_value) - WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'BGN' THEN COALESCE(c.current_value, c.signing_value) / 1.95583 - ELSE COALESCE(c.current_value, c.signing_value) * ( + CASE WHEN c.current_value IS NOT NULL THEN CASE + WHEN COALESCE((SELECT NULLIF(w.currency, '') FROM refresh_amendment_winners w + WHERE w.unp = c.unp AND w.contract_number = c.contract_number), NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN c.current_value + WHEN COALESCE((SELECT NULLIF(w.currency, '') FROM refresh_amendment_winners w + WHERE w.unp = c.unp AND w.contract_number = c.contract_number), NULLIF(c.currency, ''), 'BGN') = 'BGN' THEN c.current_value / 1.95583 + ELSE c.current_value * ( + SELECT f.eur_per_unit + FROM fx_rates f + WHERE f.base_currency = COALESCE((SELECT NULLIF(w.currency, '') FROM refresh_amendment_winners w + WHERE w.unp = c.unp AND w.contract_number = c.contract_number), NULLIF(c.currency, '')) + AND f.rate_date <= c.contract_date + AND f.rate_date >= date(c.contract_date, '-10 days') + ORDER BY f.rate_date DESC + LIMIT 1 + ) + END ELSE CASE + WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN c.signing_value + WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'BGN' THEN c.signing_value / 1.95583 + ELSE c.signing_value * ( SELECT f.eur_per_unit FROM fx_rates f WHERE f.base_currency = NULLIF(c.currency, '') @@ -960,7 +1046,7 @@ FROM ( ORDER BY f.rate_date DESC LIMIT 1 ) - END AS eff_eur, + END END AS eff_eur, CASE WHEN t.estimated_value IS NULL THEN NULL WHEN COALESCE(NULLIF(t.currency, ''), 'BGN') = 'EUR' THEN t.estimated_value @@ -1184,6 +1270,17 @@ WITH contract_base AS ( WHEN 'annex_suspect' THEN COALESCE(signing_value, current_value) ELSE COALESCE(current_value, signing_value) END AS trusted_native, + CASE new_value_flag + WHEN 'value_suspect' THEN NULL + WHEN 'annex_suspect' THEN CASE + WHEN signing_value IS NOT NULL THEN COALESCE(NULLIF(currency, ''), 'BGN') + ELSE COALESCE(NULLIF(current_value_currency, ''), NULLIF(currency, ''), 'BGN') + END + ELSE CASE + WHEN current_value IS NOT NULL THEN COALESCE(NULLIF(current_value_currency, ''), NULLIF(currency, ''), 'BGN') + ELSE COALESCE(NULLIF(currency, ''), 'BGN') + END + END AS trusted_currency, CASE WHEN new_value_flag IN ('value_suspect', 'annex_suspect') OR current_value IS NULL THEN NULL WHEN COALESCE(NULLIF(current_value_currency, ''), NULLIF(currency, ''), 'BGN') = 'EUR' THEN current_value @@ -1200,8 +1297,8 @@ WITH contract_base AS ( CASE WHEN new_value_flag = 'value_suspect' THEN proc_est_eur WHEN trusted_native IS NULL THEN NULL - WHEN COALESCE(currency, 'BGN') = 'EUR' THEN trusted_native - WHEN COALESCE(currency, 'BGN') = 'BGN' THEN trusted_native / 1.95583 + WHEN trusted_currency = 'EUR' THEN trusted_native + WHEN trusted_currency = 'BGN' THEN trusted_native / 1.95583 WHEN fx_rate IS NOT NULL THEN trusted_native * fx_rate ELSE NULL END AS new_amount_eur, @@ -1397,3 +1494,4 @@ FROM contracts c GROUP BY CASE WHEN c.eu_funded = 1 THEN '1' ELSE '0' END; DROP TABLE IF EXISTS refresh_touched_contracts; DROP TABLE IF EXISTS refresh_touched_bidders; DROP TABLE IF EXISTS refresh_touched_authorities; +DROP TABLE IF EXISTS refresh_amendment_winners; From a3acf7259294c08a68f0d48dc28477b9088b96c2 Mon Sep 17 00:00:00 2001 From: Todor Kolev Date: Mon, 20 Jul 2026 21:28:50 +0000 Subject: [PATCH 10/17] fix(etl): apply the whole migration chain to the work DB and served D1 (#245) The work-DB rebuild seeded schema from 0000_init.sql only, so a numbered migration (0002_current_value_currency) never reached it and normalize-raw aborted on the missing column. Apply every packages/db/migrations/*.sql in sorted order, which also lets 0001's index reach rebuilt work DBs for the first time. The served D1 gets the column via deploy.yml: probe the real table (wrangler's migration ledger is empty because the base schema was created out-of-band), then ALTER + backfill + precompute, gated on a completion-marker table so a partial first deploy resumes without a double ALTER. backfill-current-value-currency.sql repairs the ~2,510 already-shipped rows and self-checks parity. --- .github/workflows/deploy.yml | 69 ++++++++++++++++++ packages/db/src/migrations.test.ts | 50 +++++++++++++ scripts/backfill-current-value-currency.sql | 77 +++++++++++++++++++++ scripts/import.mjs | 16 ++++- 4 files changed, 210 insertions(+), 2 deletions(-) create mode 100644 scripts/backfill-current-value-currency.sql diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 81ba71b8..2e5789ee 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -125,6 +125,75 @@ jobs: if: steps.guard.outputs.ok == 'true' run: pnpm typecheck + # The base schema was created out-of-band with `d1 execute --file`, so wrangler's migration + # ledger is empty and `d1 migrations apply` would collide on 0000. Probe the actual table + # instead. SQLite has no `ADD COLUMN IF NOT EXISTS`; a completion-marker table is created only + # after the backfill and rollup rebuild succeed, so a partially failed first deploy safely + # resumes without trying ALTER TABLE twice. Malformed responses and impossible states are fatal. + - name: Apply and backfill amendment-value currency + if: steps.guard.outputs.ok == 'true' + run: | + node scripts/wrangler-render.mjs apps/web/wrangler.jsonc + schema_json="$(pnpm --filter @sigma/web exec wrangler d1 execute "${SIGMA_D1_NAME:-sigma}" \ + --config wrangler.deploy.jsonc --remote --yes --json \ + --command "SELECT + (SELECT COUNT(*) FROM pragma_table_info('contracts') WHERE name = 'current_value_currency') AS column_exists, + (SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'sigma_backfill_0002_current_value_currency') AS backfill_complete")" + + set +e + printf '%s' "$schema_json" | node -e ' + const fs = require("fs"); + let payload; + try { + payload = JSON.parse(fs.readFileSync(0, "utf8")); + } catch { + process.exit(2); + } + const result = Array.isArray(payload) ? payload[0] : payload; + const row = result && Array.isArray(result.results) ? result.results[0] : null; + if (!row || !Object.hasOwn(row, "column_exists") || !Object.hasOwn(row, "backfill_complete")) process.exit(2); + const column = Number(row.column_exists); + const complete = Number(row.backfill_complete); + if (column === 1 && complete === 1) process.exit(0); + if (column === 0 && complete === 0) process.exit(1); + if (column === 1 && complete === 0) process.exit(3); + process.exit(2); + ' + column_status="$?" + set -e + + repair_currency_values() { + pnpm --filter @sigma/web exec wrangler d1 execute "${SIGMA_D1_NAME:-sigma}" \ + --config wrangler.deploy.jsonc --remote --yes \ + --file ../../scripts/backfill-current-value-currency.sql + pnpm --filter @sigma/web exec wrangler d1 execute "${SIGMA_D1_NAME:-sigma}" \ + --config wrangler.deploy.jsonc --remote --yes \ + --file ../../scripts/precompute.sql + pnpm --filter @sigma/web exec wrangler d1 execute "${SIGMA_D1_NAME:-sigma}" \ + --config wrangler.deploy.jsonc --remote --yes \ + --command "CREATE TABLE sigma_backfill_0002_current_value_currency (completed_at TEXT NOT NULL DEFAULT (datetime('now')))" + } + + case "$column_status" in + 0) + echo "current_value_currency already exists; one-time migration is complete." + ;; + 1) + pnpm --filter @sigma/web exec wrangler d1 execute "${SIGMA_D1_NAME:-sigma}" \ + --config wrangler.deploy.jsonc --remote --yes \ + --file ../../packages/db/migrations/0002_current_value_currency.sql + repair_currency_values + ;; + 3) + echo "current_value_currency exists without its completion marker; resuming backfill." + repair_currency_values + ;; + *) + echo "::error::Could not determine whether current_value_currency exists." + exit 1 + ;; + esac + # `run deploy`, not `deploy` — bare `pnpm deploy` is a pnpm built-in, not our package script. - name: Deploy explorer (sigma) if: steps.guard.outputs.ok == 'true' diff --git a/packages/db/src/migrations.test.ts b/packages/db/src/migrations.test.ts index 8aa86bbf..7c4b60ea 100644 --- a/packages/db/src/migrations.test.ts +++ b/packages/db/src/migrations.test.ts @@ -10,6 +10,8 @@ const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); const migration0 = resolve(root, 'packages/db/migrations/0000_init.sql'); const migration1 = resolve(root, 'packages/db/migrations/0001_flow_pairs_bidder_index.sql'); const migration2 = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); +const backfill = resolve(root, 'scripts/backfill-current-value-currency.sql'); +const precompute = resolve(root, 'scripts/precompute.sql'); function sqlite(dbPath: string, sql: string): string { return execFileSync('sqlite3', [dbPath], { input: sql, encoding: 'utf8' }); @@ -85,4 +87,52 @@ describe('served migrations', () => { rmSync(dir, { recursive: true, force: true }); } }); + + it('backfills cross-currency amendment amounts and their rollups', () => { + const dir = mkdtempSync(resolve(tmpdir(), 'sigma-migration-backfill-')); + const dbPath = resolve(dir, 'test.sqlite'); + try { + readScript(dbPath, migration0); + readScript(dbPath, migration1); + sqlite( + dbPath, + `INSERT INTO authorities (id, name) VALUES ('auth:1', 'Authority'); + INSERT INTO bidders (id, name, kind) VALUES ('eik:1', 'Bidder', 'company'); + INSERT INTO tenders + (id, source_id, title, authority_id, cpv_code, procedure_type, status) + VALUES + ('t:UNP-1', 'UNP-1', 'Tender', 'auth:1', '45000000', 'open', 'awarded'); + INSERT INTO contracts + (id, tender_id, bidder_id, amount, currency, contract_number, signing_value, + current_value, value_flag, amount_eur, current_value_eur) + VALUES + ('c:e:1', 't:UNP-1', 'eik:1', 104748559.44, 'BGN', 'CONTRACT-1', + 136580250, 104748559.44, 'ok', 104748559.44 / 1.95583, + 104748559.44 / 1.95583); + INSERT INTO amendments + (id, natural_key, contract_number, unp, value_after, currency, published_at, source) + VALUES + ('am:1', 'am:1', 'CONTRACT-1', 'UNP-1', 104748559.44, 'EUR', + '2026-06-03', 'eop:annexes:2026-06-01');`, + ); + readScript(dbPath, migration2); + readScript(dbPath, backfill); + readScript(dbPath, precompute); + + expect( + sqlite( + dbPath, + "SELECT printf('%.2f', amount_eur) || '|' || printf('%.2f', current_value_eur) || '|' || current_value_currency FROM contracts;", + ).trim(), + ).toBe('104748559.44|104748559.44|EUR'); + expect(sqlite(dbPath, "SELECT printf('%.2f', value_eur) FROM home_totals;").trim()).toBe( + '104748559.44', + ); + expect(sqlite(dbPath, "SELECT printf('%.2f', won_eur) FROM flow_pairs;").trim()).toBe( + '104748559.44', + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); }); diff --git a/scripts/backfill-current-value-currency.sql b/scripts/backfill-current-value-currency.sql new file mode 100644 index 00000000..ff0d941e --- /dev/null +++ b/scripts/backfill-current-value-currency.sql @@ -0,0 +1,77 @@ +-- One-time served-D1 repair for 0002_current_value_currency.sql. +-- +-- The column is NULL immediately after ALTER TABLE, so derive the same winning amendment used by +-- refresh-slice.sql (latest published_at, deterministic natural-key id tiebreak), then recompute the +-- canonical amount and detail-page current value. scripts/precompute.sql runs immediately after this +-- file in deploy.yml to rebuild every rollup and search_index amount from the repaired amount_eur. +WITH paired AS ( + SELECT + c.*, + CASE WHEN c.current_value IS NOT NULL THEN COALESCE(( + SELECT NULLIF(a.currency, '') + FROM amendments a + WHERE a.unp = substr(c.tender_id, 3) + AND a.contract_number = c.contract_number + AND a.value_after IS NOT NULL + ORDER BY a.published_at DESC, a.id DESC + LIMIT 1 + ), NULLIF(c.currency, ''), 'BGN') + ELSE COALESCE(NULLIF(c.currency, ''), 'BGN') + END AS derived_current_currency + FROM contracts c +), selected AS ( + SELECT + p.*, + CASE p.value_flag + WHEN 'value_suspect' THEN NULL + WHEN 'annex_suspect' THEN COALESCE(p.signing_value, p.current_value) + ELSE COALESCE(p.current_value, p.signing_value) + END AS trusted_native, + CASE p.value_flag + WHEN 'value_suspect' THEN NULL + WHEN 'annex_suspect' THEN CASE + WHEN p.signing_value IS NOT NULL THEN COALESCE(NULLIF(p.currency, ''), 'BGN') + ELSE p.derived_current_currency + END + ELSE CASE + WHEN p.current_value IS NOT NULL THEN p.derived_current_currency + ELSE COALESCE(NULLIF(p.currency, ''), 'BGN') + END + END AS trusted_currency + FROM paired p +), repaired AS ( + SELECT + id, + derived_current_currency, + CASE + -- value_suspect is repaired from the procedure estimate upstream; do not replace that repair. + WHEN value_flag = 'value_suspect' THEN amount_eur + WHEN trusted_native IS NULL THEN NULL + WHEN trusted_currency = 'EUR' THEN trusted_native + WHEN trusted_currency = 'BGN' THEN trusted_native / 1.95583 + WHEN fx_rate IS NOT NULL THEN trusted_native * fx_rate + ELSE NULL + END AS repaired_amount_eur, + CASE + WHEN value_flag IN ('value_suspect', 'annex_suspect') OR current_value IS NULL THEN NULL + WHEN derived_current_currency = 'EUR' THEN current_value + WHEN derived_current_currency = 'BGN' THEN current_value / 1.95583 + WHEN fx_rate IS NOT NULL THEN current_value * fx_rate + ELSE NULL + END AS repaired_current_value_eur + FROM selected +) +UPDATE contracts +SET + current_value_currency = repaired.derived_current_currency, + amount_eur = repaired.repaired_amount_eur, + current_value_eur = repaired.repaired_current_value_eur +FROM repaired +WHERE repaired.id = contracts.id; + +SELECT + (SELECT COUNT(*) FROM contracts WHERE current_value_currency IS NOT NULL) AS currency_rows, + (SELECT COUNT(*) FROM contracts + WHERE value_flag = 'ok' AND current_value IS NOT NULL + AND (amount_eur IS NULL OR current_value_eur IS NULL + OR ABS(amount_eur - current_value_eur) > 0.01)) AS parity_mismatches; diff --git a/scripts/import.mjs b/scripts/import.mjs index 91ad7293..45c032c5 100644 --- a/scripts/import.mjs +++ b/scripts/import.mjs @@ -3,7 +3,15 @@ // both route through scripts/load-eop.mjs; only the date window and derive mode differ. import { execFileSync } from 'node:child_process'; -import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + writeFileSync, +} from 'node:fs'; import { basename, dirname, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { computeCatchupWindow, daysInWindow } from '../packages/ingest/src/ocds.ts'; @@ -272,7 +280,11 @@ function runWorkBackfill() { if (existsSync(workDb)) rmSync(workDb, { force: true }); console.log(`==> Sigma import (work DB ${workDb})`); - sqliteFile(workDb, resolve(root, 'packages/db/migrations/0000_init.sql')); + const migrationsDir = resolve(root, 'packages/db/migrations'); + const migrations = readdirSync(migrationsDir) + .filter((name) => /^\d+.*\.sql$/.test(name)) + .sort(); + for (const migration of migrations) sqliteFile(workDb, resolve(migrationsDir, migration)); sqliteFile(workDb, resolve(root, 'scripts/work-staging-schema.sql')); let loadFlags = explicitRangeFlags(); From a51adf98baef670109409b2aafcc906515a88ee6 Mon Sep 17 00:00:00 2001 From: Todor Kolev Date: Mon, 20 Jul 2026 23:23:31 +0000 Subject: [PATCH 11/17] fix(db): current-amount-parity must skip before precompute (#245) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit current_value_eur is populated by precompute on the served D1, not by normalize on the work DB. The integrity gate also runs on the work DB (import.mjs), where every current_value_eur is still NULL — so the parity check failed on every amended contract and aborted the rebuild. Gate it on home_totals like rollup-reconciliation: skip pre-precompute, assert on the served DB. Found by a full local rebuild; the served-DB data satisfies the invariant with 0 mismatches. --- packages/db/src/integrity-checks.test.ts | 6 ++++-- scripts/integrity-checks.mjs | 17 +++++++++++++++-- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/packages/db/src/integrity-checks.test.ts b/packages/db/src/integrity-checks.test.ts index 15b4158e..7c02e592 100644 --- a/packages/db/src/integrity-checks.test.ts +++ b/packages/db/src/integrity-checks.test.ts @@ -152,9 +152,10 @@ describe('reconciliation gate — injected violations', () => { it('current-amount-parity catches a detail/rollup EUR disagreement over one cent', () => { const db = track(freshDb()); + precompute(db); // populates home_totals so the check runs (it gates on precompute like rollup-recon) sqlite( db, - "UPDATE contracts SET current_value = 100000, current_value_eur = 100000.02 WHERE id = 'c:1';", + "UPDATE contracts SET current_value = 100000, amount_eur = 100000, current_value_eur = 100000.02 WHERE id = 'c:1';", ); const result = checkCurrentAmountParity(runner(db)); expect(result.ok).toBe(false); @@ -163,9 +164,10 @@ describe('reconciliation gate — injected violations', () => { it('current-amount-parity accepts sub-cent floating-point drift', () => { const db = track(freshDb()); + precompute(db); sqlite( db, - "UPDATE contracts SET current_value = 100000, current_value_eur = 100000.009 WHERE id = 'c:1';", + "UPDATE contracts SET current_value = 100000, amount_eur = 100000, current_value_eur = 100000.009 WHERE id = 'c:1';", ); expect(checkCurrentAmountParity(runner(db)).ok).toBe(true); }); diff --git a/scripts/integrity-checks.mjs b/scripts/integrity-checks.mjs index c3b1f598..026e51f1 100644 --- a/scripts/integrity-checks.mjs +++ b/scripts/integrity-checks.mjs @@ -152,8 +152,21 @@ export function checkRollupReconciliation(runner) { // perfectly with amount_eur. One cent is the user-visible precision boundary. export function checkCurrentAmountParity(runner) { const name = 'current-amount-parity'; - if (!tableExists(runner, 'contracts')) - return { name, ok: true, skipped: true, detail: 'contracts table absent' }; + // current_value_eur is populated by precompute.sql (served D1 / ship-domain), not by normalize on the + // work DB. Before precompute every current_value_eur is NULL, so this parity is only meaningful once + // the rollups exist — gate on home_totals exactly like rollup-reconciliation, or the work-DB gate + // aborts the whole rebuild on every amended contract. + if ( + !tableExists(runner, 'contracts') || + !tableExists(runner, 'home_totals') || + num(scalar(runner, 'SELECT COUNT(*) AS n FROM home_totals', 'n')) === 0 + ) + return { + name, + ok: true, + skipped: true, + detail: 'precompute rollups absent (current_value_eur not yet populated)', + }; const mismatches = num( scalar( runner, From 48e3302e1a46f683938fc6daff44557adbf9ac92 Mon Sep 17 00:00:00 2001 From: Todor Kolev Date: Wed, 22 Jul 2026 14:55:48 +0000 Subject: [PATCH 12/17] fix(db): exclude the unknown-bidder bucket from authority top-suppliers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit getAuthority's top-suppliers query lacked the AND b.kind <> 'unknown' guard the other three surfaces (home, companies, search_index) have, so the synthetic 'Неизвестен изпълнител' bucket could rank in an authority's public top-7. Add the same guard. --- packages/db/src/queries/details.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/db/src/queries/details.ts b/packages/db/src/queries/details.ts index 8b3604f5..960f9fbd 100644 --- a/packages/db/src/queries/details.ts +++ b/packages/db/src/queries/details.ts @@ -248,7 +248,7 @@ export async function getAuthority( .prepare( `SELECT c.bidder_id, b.name, b.kind, SUM(c.amount_eur) AS won, COUNT(*) AS n FROM contracts c JOIN tenders t ON t.id = c.tender_id JOIN bidders b ON b.id = c.bidder_id - WHERE t.authority_id = ? AND c.amount_eur IS NOT NULL + WHERE t.authority_id = ? AND c.amount_eur IS NOT NULL AND b.kind <> 'unknown' GROUP BY c.bidder_id ORDER BY won DESC LIMIT 7`, ) .bind(authorityId) From d9e291eaef111d232b0831f9d6a7a4717478e1fc Mon Sep 17 00:00:00 2001 From: Todor Kolev Date: Thu, 23 Jul 2026 13:04:30 +0000 Subject: [PATCH 13/17] fix(etl): exclude composite joint EIKs from authority minting A raw authority_eik can carry a joint-procurement composite ('EIK1; EIK2'). The rewritten canonical name/kind sources and the main authority insert consumed it verbatim, minting orphan 'auth:EIK1; EIK2' authorities referenced by no tender, contract, rollup or search row - a full 2020-2026 rebuild measured 404 of them. Guard every minting source with NOT LIKE '%;%'; joint tenders are attributed through their individual members. --- scripts/normalize-raw.sql | 11 +++++++---- scripts/refresh-slice.sql | 7 ++++--- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/scripts/normalize-raw.sql b/scripts/normalize-raw.sql index 0b4f1018..a4d49c1f 100644 --- a/scripts/normalize-raw.sql +++ b/scripts/normalize-raw.sql @@ -66,9 +66,9 @@ FROM ( FROM ( SELECT authority_eik, authority_name, COUNT(*) AS cnt FROM ( - SELECT authority_eik, authority_name FROM raw_contracts WHERE authority_eik IS NOT NULL + SELECT authority_eik, authority_name FROM raw_contracts WHERE authority_eik IS NOT NULL AND authority_eik NOT LIKE '%;%' UNION ALL - SELECT authority_eik, authority_name FROM raw_tenders WHERE authority_eik IS NOT NULL + SELECT authority_eik, authority_name FROM raw_tenders WHERE authority_eik IS NOT NULL AND authority_eik NOT LIKE '%;%' ) WHERE authority_name IS NOT NULL AND TRIM(authority_name) <> '' GROUP BY authority_eik, authority_name @@ -116,9 +116,12 @@ SELECT s.authority_eik, act.canonical_type FROM ( - SELECT authority_eik FROM raw_contracts WHERE authority_eik IS NOT NULL + -- Composite joint-procurement EIKs ('EIK1; EIK2') must not mint standalone authorities — + -- they are attributed via their individual members; an unguarded source mints orphan + -- 'auth:EIK1; EIK2' rows referenced by nothing (verified on a full 2020-2026 rebuild). + SELECT authority_eik FROM raw_contracts WHERE authority_eik IS NOT NULL AND authority_eik NOT LIKE '%;%' UNION - SELECT authority_eik FROM raw_tenders WHERE authority_eik IS NOT NULL + SELECT authority_eik FROM raw_tenders WHERE authority_eik IS NOT NULL AND authority_eik NOT LIKE '%;%' ) s LEFT JOIN authority_canonical_name acn ON acn.authority_eik = s.authority_eik LEFT JOIN authority_canonical_type act ON act.authority_eik = s.authority_eik; diff --git a/scripts/refresh-slice.sql b/scripts/refresh-slice.sql index a79a2ea7..48d5fbbd 100644 --- a/scripts/refresh-slice.sql +++ b/scripts/refresh-slice.sql @@ -46,7 +46,7 @@ FROM ( SELECT source, authority_eik, authority_name FROM raw_tenders ) WHERE (source LIKE 'eop:%' OR source LIKE 'ocds:%') - AND authority_eik IS NOT NULL + AND authority_eik IS NOT NULL AND authority_eik NOT LIKE '%;%' AND authority_name IS NOT NULL AND TRIM(authority_name) <> '' GROUP BY authority_eik, authority_name ) @@ -93,11 +93,12 @@ SELECT s.authority_eik, act.canonical_type FROM ( + -- Composite joint EIKs must not mint standalone authorities (see normalize-raw.sql). SELECT authority_eik FROM raw_contracts - WHERE (source LIKE 'eop:%' OR source LIKE 'ocds:%') AND authority_eik IS NOT NULL + WHERE (source LIKE 'eop:%' OR source LIKE 'ocds:%') AND authority_eik IS NOT NULL AND authority_eik NOT LIKE '%;%' UNION SELECT authority_eik FROM raw_tenders - WHERE (source LIKE 'eop:%' OR source LIKE 'ocds:%') AND authority_eik IS NOT NULL + WHERE (source LIKE 'eop:%' OR source LIKE 'ocds:%') AND authority_eik IS NOT NULL AND authority_eik NOT LIKE '%;%' ) s LEFT JOIN authority_canonical_name acn ON acn.authority_eik = s.authority_eik LEFT JOIN authority_canonical_type act ON act.authority_eik = s.authority_eik; From ddedd0a783d85c1db6e016e9fc6e284fc31dde5d Mon Sep 17 00:00:00 2001 From: Todor Kolev Date: Thu, 23 Jul 2026 13:04:49 +0000 Subject: [PATCH 14/17] fix(etl): skip undecomposable composite EIKs when seeding joint members A ';'-bearing member EIK is a composite the splitter could not decompose - it is not a real single authority, and member_defaults would mint an orphan 'auth:EIK1; EIK2' row from it. Guard both the normalize and refresh member inserts. --- scripts/normalize-raw.sql | 4 +++- scripts/refresh-slice.sql | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/normalize-raw.sql b/scripts/normalize-raw.sql index dc6b2639..1677841e 100644 --- a/scripts/normalize-raw.sql +++ b/scripts/normalize-raw.sql @@ -141,7 +141,9 @@ INSERT OR IGNORE INTO joint_authority_members (unp, authority_id, member_name, authority_type, source_ordinal) SELECT unp, 'auth:' || authority_eik, NULLIF(member_name, ''), authority_type, source_ordinal FROM split -WHERE authority_eik <> ''; +-- A member EIK still carrying ';' is a composite the split could not decompose - not a real +-- single authority; it must not seed a member row or mint an orphan 'auth:EIK1; EIK2'. +WHERE authority_eik <> '' AND authority_eik NOT LIKE '%;%'; -- A minority of co-authorities never occur standalone. Mint those real EIK identities from the -- positionally corresponding name component; INSERT OR IGNORE preserves every standalone row. diff --git a/scripts/refresh-slice.sql b/scripts/refresh-slice.sql index 76134d05..04826c34 100644 --- a/scripts/refresh-slice.sql +++ b/scripts/refresh-slice.sql @@ -115,7 +115,9 @@ INSERT OR IGNORE INTO refresh_joint_authority_members (unp, authority_id, member_name, authority_type, source_ordinal) SELECT unp, 'auth:' || authority_eik, NULLIF(member_name, ''), authority_type, source_ordinal FROM split -WHERE authority_eik <> ''; +-- A member EIK still carrying ';' is a composite the split could not decompose - not a real +-- single authority; it must not seed a member row or mint an orphan 'auth:EIK1; EIK2'. +WHERE authority_eik <> '' AND authority_eik NOT LIKE '%;%'; WITH name_counts AS ( SELECT authority_id, member_name, From 0a352286a0ff711bf3312ef2247cbfa00d91b033 Mon Sep 17 00:00:00 2001 From: Todor Kolev Date: Thu, 23 Jul 2026 20:03:24 +0000 Subject: [PATCH 15/17] style: prettier-format companies.test.ts --- packages/db/src/queries/companies.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/db/src/queries/companies.test.ts b/packages/db/src/queries/companies.test.ts index b0e9b9f0..f6e038e9 100644 --- a/packages/db/src/queries/companies.test.ts +++ b/packages/db/src/queries/companies.test.ts @@ -120,9 +120,9 @@ describe('streamCompaniesCsv', () => { expect(sql.filter((query) => query.includes('company_totals'))).not.toHaveLength(0); expect( - sql.filter((query) => query.includes('company_totals')).every((query) => - query.includes("kind <> 'unknown'"), - ), + sql + .filter((query) => query.includes('company_totals')) + .every((query) => query.includes("kind <> 'unknown'")), ).toBe(true); }); }); From bfd16f2713b9da8df97c5aba8243e7b099169dc8 Mon Sep 17 00:00:00 2001 From: Todor Kolev Date: Thu, 23 Jul 2026 20:11:50 +0000 Subject: [PATCH 16/17] fix(merge): carry the full fold set for the amendment-currency composition The fold diff was applied only to the six textually-conflicted files; four more files it touches (async current-amount-parity in integrity-checks.mjs, migration 0002 in the two SQL-execution tests) auto-merged to stale versions - the sync parity check against async helpers returned NaN and failed the gate. Apply the fold to them too. --- packages/db/src/contractor-identity-sql.test.ts | 5 +++++ .../db/src/etl-entity-canonicalization-sql.test.ts | 2 ++ scripts/integrity-checks.mjs | 13 ++++++------- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/packages/db/src/contractor-identity-sql.test.ts b/packages/db/src/contractor-identity-sql.test.ts index 13aa10ea..92befd79 100644 --- a/packages/db/src/contractor-identity-sql.test.ts +++ b/packages/db/src/contractor-identity-sql.test.ts @@ -7,6 +7,10 @@ import { describe, expect, it } from 'vitest'; const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); const schema = readFileSync(resolve(root, 'packages/db/migrations/0000_init.sql'), 'utf8'); +const migration2 = readFileSync( + resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'), + 'utf8', +); const staging = readFileSync(resolve(root, 'scripts/work-staging-schema.sql'), 'utf8'); const normalize = readFileSync(resolve(root, 'scripts/normalize-raw.sql'), 'utf8'); const precompute = readFileSync(resolve(root, 'scripts/precompute.sql'), 'utf8'); @@ -56,6 +60,7 @@ VALUES function build(path: 'normalize' | 'refresh'): DatabaseSync { const db = new DatabaseSync(':memory:'); db.exec(schema); + db.exec(migration2); db.exec(staging); db.exec(seed); if (path === 'normalize') { diff --git a/packages/db/src/etl-entity-canonicalization-sql.test.ts b/packages/db/src/etl-entity-canonicalization-sql.test.ts index 66bf3a17..d413cddf 100644 --- a/packages/db/src/etl-entity-canonicalization-sql.test.ts +++ b/packages/db/src/etl-entity-canonicalization-sql.test.ts @@ -8,6 +8,7 @@ import { describe, expect, it } from 'vitest'; const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); const schemaPath = resolve(root, 'packages/db/migrations/0000_init.sql'); +const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); const etlPaths = [ ['normalize-raw', resolve(root, 'scripts/normalize-raw.sql')], @@ -35,6 +36,7 @@ function withEtlDb(label: string, run: (dbPath: string) => void): void { const dbPath = resolve(dir, 'test.sqlite'); try { readScript(dbPath, schemaPath); + readScript(dbPath, migration2Path); readScript(dbPath, stagingPath); run(dbPath); } finally { diff --git a/scripts/integrity-checks.mjs b/scripts/integrity-checks.mjs index 4c989d2c..b48c4912 100644 --- a/scripts/integrity-checks.mjs +++ b/scripts/integrity-checks.mjs @@ -157,16 +157,15 @@ export async function checkRollupReconciliation(runner) { // current_value_eur while every aggregate and search result reads amount_eur, so allowing these two // columns to disagree makes the served database contradict itself even when each rollup reconciles // perfectly with amount_eur. One cent is the user-visible precision boundary. -export function checkCurrentAmountParity(runner) { +export async function checkCurrentAmountParity(runner) { const name = 'current-amount-parity'; // current_value_eur is populated by precompute.sql (served D1 / ship-domain), not by normalize on the // work DB. Before precompute every current_value_eur is NULL, so this parity is only meaningful once - // the rollups exist — gate on home_totals exactly like rollup-reconciliation, or the work-DB gate - // aborts the whole rebuild on every amended contract. + // the rollups exist — gate on home_totals exactly like rollup-reconciliation. if ( - !tableExists(runner, 'contracts') || - !tableExists(runner, 'home_totals') || - num(scalar(runner, 'SELECT COUNT(*) AS n FROM home_totals', 'n')) === 0 + !(await tableExists(runner, 'contracts')) || + !(await tableExists(runner, 'home_totals')) || + num(await scalar(runner, 'SELECT COUNT(*) AS n FROM home_totals', 'n')) === 0 ) return { name, @@ -175,7 +174,7 @@ export function checkCurrentAmountParity(runner) { detail: 'precompute rollups absent (current_value_eur not yet populated)', }; const mismatches = num( - scalar( + await scalar( runner, "SELECT COUNT(*) AS n FROM contracts WHERE value_flag = 'ok' AND current_value IS NOT NULL " + 'AND (amount_eur IS NULL OR current_value_eur IS NULL OR ABS(amount_eur - current_value_eur) > 0.01)', From 30eea1ead28b0a1fa555ba492bca18374d501e15 Mon Sep 17 00:00:00 2001 From: Todor Kolev Date: Thu, 23 Jul 2026 20:15:16 +0000 Subject: [PATCH 17/17] test(etl): teach the gate fixture about current-amount-parity #261 adds current-amount-parity to the shared roster; the fakeD1 from #156 did not answer its query (NaN) and the skip-count expectation assumed only rollup-reconciliation un-skips with rollups present. Answer the parity query clean and expect two fewer skips. --- apps/etl/src/integrity.test.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/apps/etl/src/integrity.test.ts b/apps/etl/src/integrity.test.ts index db439ffe..91cac264 100644 --- a/apps/etl/src/integrity.test.ts +++ b/apps/etl/src/integrity.test.ts @@ -62,6 +62,10 @@ function fakeD1(seed: Seed = {}): D1Database { } if (sql.includes('spent_eur < 0')) return [{ a: 0, c: 0, f: 0 }]; if (sql.includes('signed_at')) return [{ n: seed.badDates ?? 0 }]; + if (sql.includes('current_value_eur')) { + // current-amount-parity (#261): clean by default — no detail/rollup disagreement. + return [{ n: 0 }]; + } if (sql.includes('FROM contracts') && sql.includes('COUNT(*) AS n')) { return [{ n: seed.contracts ?? 5 }]; } @@ -127,9 +131,9 @@ describe('runServedIntegrityGate', () => { const ok = log.events.find((e) => e.event.event === 'etl_integrity_ok'); return ok?.event.skipped as number; }; - // With home_totals present, one fewer check self-skips than on the bare work DB — that check is - // rollup-reconciliation moving from skipped to actually run over the live D1. - expect(await skippedFor({ rollups: true })).toBe((await skippedFor({})) - 1); + // With home_totals present, two fewer checks self-skip than on the bare work DB — + // rollup-reconciliation and current-amount-parity (#261) both move from skipped to run. + expect(await skippedFor({ rollups: true })).toBe((await skippedFor({})) - 2); }); it('throws when a rollup no longer reconciles with SUM(amount_eur) over the live D1', async () => {