diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 81ba71b8..2e5789ee 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -125,6 +125,75 @@ jobs: if: steps.guard.outputs.ok == 'true' run: pnpm typecheck + # The base schema was created out-of-band with `d1 execute --file`, so wrangler's migration + # ledger is empty and `d1 migrations apply` would collide on 0000. Probe the actual table + # instead. SQLite has no `ADD COLUMN IF NOT EXISTS`; a completion-marker table is created only + # after the backfill and rollup rebuild succeed, so a partially failed first deploy safely + # resumes without trying ALTER TABLE twice. Malformed responses and impossible states are fatal. + - name: Apply and backfill amendment-value currency + if: steps.guard.outputs.ok == 'true' + run: | + node scripts/wrangler-render.mjs apps/web/wrangler.jsonc + schema_json="$(pnpm --filter @sigma/web exec wrangler d1 execute "${SIGMA_D1_NAME:-sigma}" \ + --config wrangler.deploy.jsonc --remote --yes --json \ + --command "SELECT + (SELECT COUNT(*) FROM pragma_table_info('contracts') WHERE name = 'current_value_currency') AS column_exists, + (SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'sigma_backfill_0002_current_value_currency') AS backfill_complete")" + + set +e + printf '%s' "$schema_json" | node -e ' + const fs = require("fs"); + let payload; + try { + payload = JSON.parse(fs.readFileSync(0, "utf8")); + } catch { + process.exit(2); + } + const result = Array.isArray(payload) ? payload[0] : payload; + const row = result && Array.isArray(result.results) ? result.results[0] : null; + if (!row || !Object.hasOwn(row, "column_exists") || !Object.hasOwn(row, "backfill_complete")) process.exit(2); + const column = Number(row.column_exists); + const complete = Number(row.backfill_complete); + if (column === 1 && complete === 1) process.exit(0); + if (column === 0 && complete === 0) process.exit(1); + if (column === 1 && complete === 0) process.exit(3); + process.exit(2); + ' + column_status="$?" + set -e + + repair_currency_values() { + pnpm --filter @sigma/web exec wrangler d1 execute "${SIGMA_D1_NAME:-sigma}" \ + --config wrangler.deploy.jsonc --remote --yes \ + --file ../../scripts/backfill-current-value-currency.sql + pnpm --filter @sigma/web exec wrangler d1 execute "${SIGMA_D1_NAME:-sigma}" \ + --config wrangler.deploy.jsonc --remote --yes \ + --file ../../scripts/precompute.sql + pnpm --filter @sigma/web exec wrangler d1 execute "${SIGMA_D1_NAME:-sigma}" \ + --config wrangler.deploy.jsonc --remote --yes \ + --command "CREATE TABLE sigma_backfill_0002_current_value_currency (completed_at TEXT NOT NULL DEFAULT (datetime('now')))" + } + + case "$column_status" in + 0) + echo "current_value_currency already exists; one-time migration is complete." + ;; + 1) + pnpm --filter @sigma/web exec wrangler d1 execute "${SIGMA_D1_NAME:-sigma}" \ + --config wrangler.deploy.jsonc --remote --yes \ + --file ../../packages/db/migrations/0002_current_value_currency.sql + repair_currency_values + ;; + 3) + echo "current_value_currency exists without its completion marker; resuming backfill." + repair_currency_values + ;; + *) + echo "::error::Could not determine whether current_value_currency exists." + exit 1 + ;; + esac + # `run deploy`, not `deploy` — bare `pnpm deploy` is a pnpm built-in, not our package script. - name: Deploy explorer (sigma) if: steps.guard.outputs.ok == 'true' diff --git a/apps/etl/src/integrity.test.ts b/apps/etl/src/integrity.test.ts index db439ffe..91cac264 100644 --- a/apps/etl/src/integrity.test.ts +++ b/apps/etl/src/integrity.test.ts @@ -62,6 +62,10 @@ function fakeD1(seed: Seed = {}): D1Database { } if (sql.includes('spent_eur < 0')) return [{ a: 0, c: 0, f: 0 }]; if (sql.includes('signed_at')) return [{ n: seed.badDates ?? 0 }]; + if (sql.includes('current_value_eur')) { + // current-amount-parity (#261): clean by default — no detail/rollup disagreement. + return [{ n: 0 }]; + } if (sql.includes('FROM contracts') && sql.includes('COUNT(*) AS n')) { return [{ n: seed.contracts ?? 5 }]; } @@ -127,9 +131,9 @@ describe('runServedIntegrityGate', () => { const ok = log.events.find((e) => e.event.event === 'etl_integrity_ok'); return ok?.event.skipped as number; }; - // With home_totals present, one fewer check self-skips than on the bare work DB — that check is - // rollup-reconciliation moving from skipped to actually run over the live D1. - expect(await skippedFor({ rollups: true })).toBe((await skippedFor({})) - 1); + // With home_totals present, two fewer checks self-skip than on the bare work DB — + // rollup-reconciliation and current-amount-parity (#261) both move from skipped to run. + expect(await skippedFor({ rollups: true })).toBe((await skippedFor({})) - 2); }); it('throws when a rollup no longer reconciles with SUM(amount_eur) over the live D1', async () => { diff --git a/packages/db/migrations/0002_current_value_currency.sql b/packages/db/migrations/0002_current_value_currency.sql new file mode 100644 index 00000000..03f43420 --- /dev/null +++ b/packages/db/migrations/0002_current_value_currency.sql @@ -0,0 +1,6 @@ +-- Track the currency of whichever amendment last set contracts.current_value. +-- current_value can be denominated in a DIFFERENT currency than the contract's +-- original signing currency (contracts.currency) when the latest amendment was +-- recorded after ЦАИС ЕОП's 2026 BGN->EUR feed switch. EUR conversions of +-- current_value must use this column, not contracts.currency. +ALTER TABLE contracts ADD COLUMN current_value_currency TEXT; diff --git a/packages/db/src/contractor-identity-sql.test.ts b/packages/db/src/contractor-identity-sql.test.ts index 13aa10ea..92befd79 100644 --- a/packages/db/src/contractor-identity-sql.test.ts +++ b/packages/db/src/contractor-identity-sql.test.ts @@ -7,6 +7,10 @@ import { describe, expect, it } from 'vitest'; const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); const schema = readFileSync(resolve(root, 'packages/db/migrations/0000_init.sql'), 'utf8'); +const migration2 = readFileSync( + resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'), + 'utf8', +); const staging = readFileSync(resolve(root, 'scripts/work-staging-schema.sql'), 'utf8'); const normalize = readFileSync(resolve(root, 'scripts/normalize-raw.sql'), 'utf8'); const precompute = readFileSync(resolve(root, 'scripts/precompute.sql'), 'utf8'); @@ -56,6 +60,7 @@ VALUES function build(path: 'normalize' | 'refresh'): DatabaseSync { const db = new DatabaseSync(':memory:'); db.exec(schema); + db.exec(migration2); db.exec(staging); db.exec(seed); if (path === 'normalize') { diff --git a/packages/db/src/etl-entity-canonicalization-sql.test.ts b/packages/db/src/etl-entity-canonicalization-sql.test.ts index 66bf3a17..d413cddf 100644 --- a/packages/db/src/etl-entity-canonicalization-sql.test.ts +++ b/packages/db/src/etl-entity-canonicalization-sql.test.ts @@ -8,6 +8,7 @@ import { describe, expect, it } from 'vitest'; const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); const schemaPath = resolve(root, 'packages/db/migrations/0000_init.sql'); +const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); const etlPaths = [ ['normalize-raw', resolve(root, 'scripts/normalize-raw.sql')], @@ -35,6 +36,7 @@ function withEtlDb(label: string, run: (dbPath: string) => void): void { const dbPath = resolve(dir, 'test.sqlite'); try { readScript(dbPath, schemaPath); + readScript(dbPath, migration2Path); readScript(dbPath, stagingPath); run(dbPath); } finally { diff --git a/packages/db/src/integrity-checks.test.ts b/packages/db/src/integrity-checks.test.ts index ffbf94f3..04b4cbdd 100644 --- a/packages/db/src/integrity-checks.test.ts +++ b/packages/db/src/integrity-checks.test.ts @@ -13,6 +13,7 @@ import { fileURLToPath } from 'node:url'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { assertIntegrity, + checkCurrentAmountParity, checkDateSanity, checkEikValidity, checkNonEmptyCorpus, @@ -23,6 +24,8 @@ import { const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); const schemaPath = resolve(root, 'packages/db/migrations/0000_init.sql'); +const migration1Path = resolve(root, 'packages/db/migrations/0001_flow_pairs_bidder_index.sql'); +const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); const precomputePath = resolve(root, 'scripts/precompute.sql'); function sqlite(dbPath: string, sql: string): void { @@ -63,6 +66,8 @@ function freshDb(): string { const dir = mkdtempSync(resolve(tmpdir(), 'sigma-integrity-')); const dbPath = resolve(dir, 'test.sqlite'); readScript(dbPath, schemaPath); + readScript(dbPath, migration1Path); + readScript(dbPath, migration2Path); sqlite(dbPath, CLEAN_FIXTURE); return dbPath; } @@ -104,6 +109,7 @@ describe('reconciliation gate — clean corpus', () => { for (const nm of [ 'non-empty-corpus', 'rollup-reconciliation', + 'current-amount-parity', 'no-negative-values', 'eik-validity', 'date-sanity', @@ -158,6 +164,28 @@ describe('reconciliation gate — injected violations', () => { expect(result.detail).toMatch(/orphan|unattributed/); }); + it('current-amount-parity catches a detail/rollup EUR disagreement over one cent', async () => { + const db = track(freshDb()); + precompute(db); // populates home_totals so the check runs (it gates on precompute like rollup-recon) + sqlite( + db, + "UPDATE contracts SET current_value = 100000, amount_eur = 100000, current_value_eur = 100000.02 WHERE id = 'c:1';", + ); + const result = await checkCurrentAmountParity(runner(db)); + expect(result.ok).toBe(false); + expect(result.detail).toMatch(/1 ok contract.*amount_eur != current_value_eur/); + }); + + it('current-amount-parity accepts sub-cent floating-point drift', async () => { + const db = track(freshDb()); + precompute(db); + sqlite( + db, + "UPDATE contracts SET current_value = 100000, amount_eur = 100000, current_value_eur = 100000.009 WHERE id = 'c:1';", + ); + expect((await checkCurrentAmountParity(runner(db))).ok).toBe(true); + }); + it('no-negative-values catches a negative ok amount_eur (Sigma derivation bug → hard fail)', async () => { const db = track(freshDb()); sqlite(db, "UPDATE contracts SET amount_eur = -100 WHERE id = 'c:1';"); diff --git a/packages/db/src/migrations.test.ts b/packages/db/src/migrations.test.ts index f0fcb985..72e4e48b 100644 --- a/packages/db/src/migrations.test.ts +++ b/packages/db/src/migrations.test.ts @@ -9,6 +9,9 @@ import { describe, expect, it } from 'vitest'; const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); const migration0 = resolve(root, 'packages/db/migrations/0000_init.sql'); const migration1 = resolve(root, 'packages/db/migrations/0001_flow_pairs_bidder_index.sql'); +const migration2 = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); +const backfill = resolve(root, 'scripts/backfill-current-value-currency.sql'); +const precompute = resolve(root, 'scripts/precompute.sql'); function sqlite(dbPath: string, sql: string): string { return execFileSync('sqlite3', [dbPath], { input: sql, encoding: 'utf8' }); @@ -27,6 +30,7 @@ describe('served migrations', () => { try { readScript(dbPath, migration0); readScript(dbPath, migration1); + readScript(dbPath, migration2); expect( sqlite( @@ -85,6 +89,61 @@ describe('served migrations', () => { expect( sqlite(dbPath, "SELECT COUNT(*) FROM sqlite_master WHERE name LIKE 'raw_%';").trim(), ).toBe('0'); + + expect( + sqlite( + dbPath, + "SELECT COUNT(*) FROM pragma_table_info('contracts') WHERE name='current_value_currency';", + ).trim(), + ).toBe('1'); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('backfills cross-currency amendment amounts and their rollups', () => { + const dir = mkdtempSync(resolve(tmpdir(), 'sigma-migration-backfill-')); + const dbPath = resolve(dir, 'test.sqlite'); + try { + readScript(dbPath, migration0); + readScript(dbPath, migration1); + sqlite( + dbPath, + `INSERT INTO authorities (id, name) VALUES ('auth:1', 'Authority'); + INSERT INTO bidders (id, name, kind) VALUES ('eik:1', 'Bidder', 'company'); + INSERT INTO tenders + (id, source_id, title, authority_id, cpv_code, procedure_type, status) + VALUES + ('t:UNP-1', 'UNP-1', 'Tender', 'auth:1', '45000000', 'open', 'awarded'); + INSERT INTO contracts + (id, tender_id, bidder_id, amount, currency, contract_number, signing_value, + current_value, value_flag, amount_eur, current_value_eur) + VALUES + ('c:e:1', 't:UNP-1', 'eik:1', 104748559.44, 'BGN', 'CONTRACT-1', + 136580250, 104748559.44, 'ok', 104748559.44 / 1.95583, + 104748559.44 / 1.95583); + INSERT INTO amendments + (id, natural_key, contract_number, unp, value_after, currency, published_at, source) + VALUES + ('am:1', 'am:1', 'CONTRACT-1', 'UNP-1', 104748559.44, 'EUR', + '2026-06-03', 'eop:annexes:2026-06-01');`, + ); + readScript(dbPath, migration2); + readScript(dbPath, backfill); + readScript(dbPath, precompute); + + expect( + sqlite( + dbPath, + "SELECT printf('%.2f', amount_eur) || '|' || printf('%.2f', current_value_eur) || '|' || current_value_currency FROM contracts;", + ).trim(), + ).toBe('104748559.44|104748559.44|EUR'); + expect(sqlite(dbPath, "SELECT printf('%.2f', value_eur) FROM home_totals;").trim()).toBe( + '104748559.44', + ); + expect(sqlite(dbPath, "SELECT printf('%.2f', won_eur) FROM flow_pairs;").trim()).toBe( + '104748559.44', + ); } finally { rmSync(dir, { recursive: true, force: true }); } diff --git a/packages/db/src/queries/details.ts b/packages/db/src/queries/details.ts index 753db157..eb7fe5d3 100644 --- a/packages/db/src/queries/details.ts +++ b/packages/db/src/queries/details.ts @@ -393,6 +393,7 @@ interface ContractDetailRow { subcontractor_name: string | null; subcontract_value: number | null; contract_currency: string; + current_value_currency: string | null; // tender title: string; unp: string; @@ -470,6 +471,7 @@ export async function getContract( c.bids_received, c.bids_rejected, c.bids_sme, c.bids_non_eea, c.subcontractor_eik, c.subcontractor_name, c.subcontract_value, c.currency AS contract_currency, c.ordering_unit_name AS source_authority_name, + c.current_value_currency, t.title, t.source_id AS unp, t.procedure_type, t.cpv_code, t.cpv_description, t.num_lots, t.eop_tender_id, t.estimated_value, t.currency AS tender_currency, t.start_date, t.end_date, @@ -537,7 +539,8 @@ export async function getContract( const signingEur = r.signing_value_eur ?? eurFromNative(r.signing_value, r.contract_currency, r.fx_rate); const currentRaw = - r.current_value_eur ?? eurFromNative(r.current_value, r.contract_currency, r.fx_rate); + r.current_value_eur ?? + eurFromNative(r.current_value, r.current_value_currency || r.contract_currency, r.fx_rate); const procedureEstimatedEur = eurFromNative( r.estimated_value, r.tender_currency, diff --git a/packages/db/src/refresh-slice.test.ts b/packages/db/src/refresh-slice.test.ts index c64b3e38..6ee7471f 100644 --- a/packages/db/src/refresh-slice.test.ts +++ b/packages/db/src/refresh-slice.test.ts @@ -9,8 +9,12 @@ import { assertIntegrity } from '../../../scripts/integrity-checks.mjs'; const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); const schemaPath = resolve(root, 'packages/db/migrations/0000_init.sql'); +const migration1Path = resolve(root, 'packages/db/migrations/0001_flow_pairs_bidder_index.sql'); +const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); const refreshSlicePath = resolve(root, 'scripts/refresh-slice.sql'); const normalizePath = resolve(root, 'scripts/normalize-raw.sql'); +const deriveAmendmentsPath = resolve(root, 'scripts/derive-amendments.sql'); +const promoteAmendmentsPath = resolve(root, 'scripts/promote-amendments.sql'); const precomputePath = resolve(root, 'scripts/precompute.sql'); const workStagingSchemaPath = resolve(root, 'scripts/work-staging-schema.sql'); @@ -176,9 +180,82 @@ function seedOcdsOnlySharedNumber(dbPath: string): void { function initWorkDb(dbPath: string): void { readScript(dbPath, schemaPath); + readScript(dbPath, migration1Path); + readScript(dbPath, migration2Path); readScript(dbPath, workStagingSchemaPath); } +function seedCrossCurrencyAmendment(dbPath: string): void { + sqlite( + dbPath, + `INSERT INTO raw_tenders + (source, dataset_year, fetched_at, unp, tender_id, procedure_type, procurement_subject, + cpv_code, cpv_description, contract_kind, estimated_value, currency, authority_name, + authority_eik, authority_type, published_at) + VALUES + ('eop:tenders:2025-06-01', 2025, '2026-06-08T00:00:00Z', 'UNP-CROSS-CURRENCY', + 'TENDER-CROSS-CURRENCY', 'open', 'Cross-currency tender', '45000000', 'Construction', + 'works', 300000000, 'EUR', 'Authority Cross Currency', '133456789', 'public', + '2025-06-01'); + + INSERT INTO raw_contracts + (source, dataset_year, dataset_variant, fetched_at, needs_enrichment, document_number, + published_at, unp, tender_ext_id, procedure_type, procurement_subject, cpv_code, + cpv_description, contract_kind, estimated_value, procurement_currency, authority_name, + authority_eik, authority_type, contract_number, contract_date, signing_value, currency, + contract_subject, awarded_to_group, contractor_eik, contractor_name) + VALUES + ('eop:contracts:2025-06-01', 2025, 'eop', '2026-06-08T00:00:00Z', 0, + 'DOC-CROSS-CURRENCY', '2025-06-01', 'UNP-CROSS-CURRENCY', 'TENDER-CROSS-CURRENCY', + 'open', 'Cross-currency tender', '45000000', 'Construction', 'works', 300000000, + 'EUR', 'Authority Cross Currency', '133456789', 'public', 'CONTRACT-CROSS-CURRENCY', + '2025-06-02', 136580250, 'BGN', 'Cross-currency contract', 0, '997654321', + 'Bidder Cross Currency'); + + INSERT INTO raw_amendments + (source, dataset_year, dataset_variant, fetched_at, seq_no, document_number, + contract_number, contract_date, published_at, unp, authority_eik, authority_name, + procurement_subject, contract_kind, value_before, value_after, value_delta, currency, + description) + VALUES + ('eop:annexes:2026-06-01', 2026, 'eop', '2026-06-08T00:00:00Z', '1', + 'AMD-CROSS-CURRENCY', 'CONTRACT-CROSS-CURRENCY', '2025-06-02', '2026-06-03', + 'UNP-CROSS-CURRENCY', '133456789', 'Authority Cross Currency', 'Cross-currency tender', + 'works', 136580250, 104748559.44, -31831690.56, 'EUR', + 'Post-switch EUR amendment');`, + ); +} + +function seedCrossCurrencyFlagBoundary(dbPath: string): void { + sqlite( + dbPath, + `INSERT INTO raw_tenders + (source, dataset_year, fetched_at, unp, tender_id, procedure_type, procurement_subject, + cpv_code, estimated_value, currency, authority_name, authority_eik, authority_type) + VALUES + ('eop:tenders:2025-06-01', 2025, '2026-06-08T00:00:00Z', 'UNP-FX-FLAG', + 'TENDER-FX-FLAG', 'open', 'Currency flag tender', '45000000', 1500, 'EUR', + 'Authority Currency Flag', '143456789', 'public'); + INSERT INTO raw_contracts + (source, dataset_year, dataset_variant, fetched_at, needs_enrichment, unp, tender_ext_id, + procedure_type, procurement_subject, cpv_code, estimated_value, procurement_currency, + authority_name, authority_eik, authority_type, contract_number, contract_date, signing_value, + currency, contract_subject, awarded_to_group, contractor_eik, contractor_name) + VALUES + ('eop:contracts:2025-06-01', 2025, 'eop', '2026-06-08T00:00:00Z', 0, + 'UNP-FX-FLAG', 'TENDER-FX-FLAG', 'open', 'Currency flag tender', '45000000', 1500, + 'EUR', 'Authority Currency Flag', '143456789', 'public', 'CONTRACT-FX-FLAG', + '2025-06-02', 10000, 'BGN', 'Currency flag contract', 0, '987654322', + 'Bidder Currency Flag'); + INSERT INTO raw_amendments + (source, dataset_year, dataset_variant, fetched_at, document_number, contract_number, + published_at, unp, value_before, value_after, value_delta, currency) + VALUES + ('eop:annexes:2026-06-01', 2026, 'eop', '2026-06-08T00:00:00Z', 'AMD-FX-FLAG', + 'CONTRACT-FX-FLAG', '2026-06-03', 'UNP-FX-FLAG', 10000, 20000, 10000, 'EUR');`, + ); +} + function seedContractIdFixture(dbPath: string): void { sqlite( dbPath, @@ -493,6 +570,8 @@ describe('refresh-slice EOP base derivation', () => { const dbPath = resolve(dir, 'test.sqlite'); try { readScript(dbPath, schemaPath); + readScript(dbPath, migration1Path); + readScript(dbPath, migration2Path); readScript(dbPath, workStagingSchemaPath); seedEopBaseDay(dbPath); @@ -572,6 +651,8 @@ describe('refresh-slice EOP base derivation', () => { const dbPath = resolve(dir, 'test.sqlite'); try { readScript(dbPath, schemaPath); + readScript(dbPath, migration1Path); + readScript(dbPath, migration2Path); readScript(dbPath, workStagingSchemaPath); seedEopOnlySharedNumber(dbPath); readScript(dbPath, refreshSlicePath); @@ -620,6 +701,8 @@ describe('refresh-slice EOP base derivation', () => { const dbPath = resolve(dir, 'test.sqlite'); try { readScript(dbPath, schemaPath); + readScript(dbPath, migration1Path); + readScript(dbPath, migration2Path); readScript(dbPath, workStagingSchemaPath); sqlite( dbPath, @@ -663,11 +746,149 @@ describe('refresh-slice EOP base derivation', () => { } }); + it('converts current_value_eur from the amendment currency, not the contract signing currency (#245)', () => { + const dir = mkdtempSync(resolve(tmpdir(), 'sigma-refresh-slice-')); + const dbPath = resolve(dir, 'test.sqlite'); + try { + readScript(dbPath, schemaPath); + readScript(dbPath, migration1Path); + readScript(dbPath, migration2Path); + readScript(dbPath, workStagingSchemaPath); + sqlite( + dbPath, + `INSERT INTO authorities (id, name, bulstat, type) VALUES ('auth:523456789', 'Authority Eur', '523456789', 'public'); + INSERT INTO bidders (id, name, bulstat, eik_normalized, eik_valid, kind) VALUES ('eik:877777777', 'Bidder Eur', '877777777', '877777777', 1, 'company'); + INSERT INTO tenders (id, source_id, title, authority_id, estimated_value, currency, procedure_type, status) + VALUES ('t:UNP-EUR', 'UNP-EUR', 'Eur tender', 'auth:523456789', 5000, 'BGN', 'open', 'awarded'); + INSERT INTO contracts + (id, tender_id, bidder_id, amount, currency, signed_at, contract_number, signing_value, + current_value, current_value_currency, annex_count, value_flag, amount_eur, signing_value_eur, current_value_eur) + VALUES + ('c:e:eurannex', 't:UNP-EUR', 'eik:877777777', 500, 'BGN', '2025-06-02', + 'CONTRACT-EUR', 1000, 500, 'BGN', 1, 'ok', 500 / 1.95583, 1000 / 1.95583, 500 / 1.95583); + INSERT INTO raw_amendments + (source, dataset_year, dataset_variant, fetched_at, seq_no, document_number, + contract_number, contract_date, published_at, unp, authority_eik, authority_name, + procurement_subject, contract_kind, value_before, value_after, value_delta, + currency, description) + VALUES + ('eop:annexes:2026-06-02', 2026, 'eop', '2026-06-08T00:00:00Z', '2', 'AMD-EUR-2', + 'CONTRACT-EUR', '2026-06-02', '2026-06-03', 'UNP-EUR', '523456789', 'Authority Eur', + 'Eur tender', 'works', 500, 2000, 1500, 'EUR', 'Post-switch EUR annex');`, + ); + + readScript(dbPath, refreshSlicePath); + const row = sqliteJson<{ + value_flag: string; + current_value: number; + current_value_currency: string; + amount_eur: number; + current_value_eur: number; + signing_value_eur: number; + }>( + dbPath, + `SELECT value_flag, current_value, current_value_currency, amount_eur, current_value_eur, signing_value_eur + FROM contracts WHERE id = 'c:e:eurannex'`, + )[0]; + expect(row?.value_flag).toBe('ok'); + expect(row?.current_value).toBe(2000); + expect(row?.current_value_currency).toBe('EUR'); + // The amendment's own EUR value, unconverted — NOT divided by the BGN peg a second time. + expect(row?.amount_eur).toBeCloseTo(2000, 6); + expect(row?.current_value_eur).toBeCloseTo(2000, 6); + expect( + sqliteJson<{ won_eur: number }>( + dbPath, + "SELECT won_eur FROM company_totals WHERE bidder_id = 'eik:877777777'", + )[0]?.won_eur, + ).toBeCloseTo(2000, 6); + // signing_value stays denominated in the contract's own (BGN) signing currency — unaffected. + expect(row?.signing_value_eur).toBeCloseTo(1000 / 1.95583, 6); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('normalizes amount_eur and full rollups from a non-BGN amendment currency', () => { + const dir = mkdtempSync(resolve(tmpdir(), 'sigma-normalize-currency-')); + const dbPath = resolve(dir, 'test.sqlite'); + try { + initWorkDb(dbPath); + seedCrossCurrencyAmendment(dbPath); + readScript(dbPath, deriveAmendmentsPath); + readScript(dbPath, normalizePath); + readScript(dbPath, promoteAmendmentsPath); + readScript(dbPath, precomputePath); + + const expected = 104748559.44; + const contract = sqliteJson<{ + value_flag: string; + current_value_currency: string; + amount_eur: number; + current_value_eur: number; + }>( + dbPath, + `SELECT value_flag, current_value_currency, amount_eur, current_value_eur + FROM contracts WHERE contract_number = 'CONTRACT-CROSS-CURRENCY'`, + )[0]; + expect(contract?.value_flag).toBe('ok'); + expect(contract?.current_value_currency).toBe('EUR'); + expect(contract?.amount_eur).toBeCloseTo(expected, 2); + expect(contract?.current_value_eur).toBeCloseTo(expected, 2); + + for (const [table, column] of [ + ['company_totals', 'won_eur'], + ['authority_totals', 'spent_eur'], + ['sector_totals', 'value_eur'], + ['home_totals', 'value_eur'], + ['flow_pairs', 'won_eur'], + ] as const) { + expect( + sqliteJson<{ total: number }>(dbPath, `SELECT ${column} AS total FROM ${table}`)[0] + ?.total, + `${table}.${column}`, + ).toBeCloseTo(expected, 2); + } + expect( + sqliteJson<{ amount: number }>( + dbPath, + "SELECT amount FROM search_index WHERE kind = 'contract'", + )[0]?.amount, + ).toBeCloseTo(expected, 2); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('classifies cross-currency amendments identically in full and slice derives', () => { + const dir = mkdtempSync(resolve(tmpdir(), 'sigma-currency-flag-parity-')); + const fullDb = resolve(dir, 'full.sqlite'); + const sliceDb = resolve(dir, 'slice.sqlite'); + try { + for (const dbPath of [fullDb, sliceDb]) { + initWorkDb(dbPath); + seedCrossCurrencyFlagBoundary(dbPath); + readScript(dbPath, deriveAmendmentsPath); + } + readScript(fullDb, normalizePath); + readScript(sliceDb, refreshSlicePath); + + const valueSql = `SELECT value_flag, ROUND(amount_eur, 2) AS amount_eur + FROM contracts WHERE contract_number = 'CONTRACT-FX-FLAG'`; + expect(sqliteJson(fullDb, valueSql)).toEqual([{ value_flag: 'review', amount_eur: 20000 }]); + expect(sqliteJson(sliceDb, valueSql)).toEqual(sqliteJson(fullDb, valueSql)); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + it('classifies amendment rollups from the contract-row estimated value', () => { const dir = mkdtempSync(resolve(tmpdir(), 'sigma-refresh-slice-')); const dbPath = resolve(dir, 'test.sqlite'); try { readScript(dbPath, schemaPath); + readScript(dbPath, migration1Path); + readScript(dbPath, migration2Path); readScript(dbPath, workStagingSchemaPath); sqlite( dbPath, @@ -705,7 +926,7 @@ describe('refresh-slice EOP base derivation', () => { VALUES ('eop:annexes:2026-06-01', 2026, 'eop', '2026-06-07T00:00:00Z', '1', 'AMD-MISMATCH', 'CONTRACT-MISMATCH', '2026-06-02', '2026-06-03', 'UNP-MISMATCH', '423456789', 'Authority Mismatch', - 'Mismatch tender', 'works', 1000, 500000, 499000, 'BGN', 'Huge increase');`, + 'Mismatch tender', 'works', 1000, 500000, 499000, 'EUR', 'Huge increase');`, ); readScript(dbPath, refreshSlicePath); @@ -713,13 +934,16 @@ describe('refresh-slice EOP base derivation', () => { value_flag: string; amount: number; amount_eur: number | null; + current_value_currency: string; signing_value_eur: number | null; }>( dbPath, - "SELECT value_flag, amount, amount_eur, signing_value_eur FROM contracts WHERE contract_number = 'CONTRACT-MISMATCH'", + "SELECT value_flag, amount, amount_eur, current_value_currency, signing_value_eur FROM contracts WHERE contract_number = 'CONTRACT-MISMATCH'", )[0]; expect(row?.value_flag).toBe('annex_suspect'); expect(row?.amount).toBe(1000); + expect(row?.current_value_currency).toBe('EUR'); + // The suspect EUR amendment is rejected, so the selected signing_value still converts as BGN. expect(row?.amount_eur).toBeCloseTo(1000 / 1.95583, 6); expect(row?.signing_value_eur).toBeCloseTo(1000 / 1.95583, 6); } finally { diff --git a/scripts/backfill-current-value-currency.sql b/scripts/backfill-current-value-currency.sql new file mode 100644 index 00000000..ff0d941e --- /dev/null +++ b/scripts/backfill-current-value-currency.sql @@ -0,0 +1,77 @@ +-- One-time served-D1 repair for 0002_current_value_currency.sql. +-- +-- The column is NULL immediately after ALTER TABLE, so derive the same winning amendment used by +-- refresh-slice.sql (latest published_at, deterministic natural-key id tiebreak), then recompute the +-- canonical amount and detail-page current value. scripts/precompute.sql runs immediately after this +-- file in deploy.yml to rebuild every rollup and search_index amount from the repaired amount_eur. +WITH paired AS ( + SELECT + c.*, + CASE WHEN c.current_value IS NOT NULL THEN COALESCE(( + SELECT NULLIF(a.currency, '') + FROM amendments a + WHERE a.unp = substr(c.tender_id, 3) + AND a.contract_number = c.contract_number + AND a.value_after IS NOT NULL + ORDER BY a.published_at DESC, a.id DESC + LIMIT 1 + ), NULLIF(c.currency, ''), 'BGN') + ELSE COALESCE(NULLIF(c.currency, ''), 'BGN') + END AS derived_current_currency + FROM contracts c +), selected AS ( + SELECT + p.*, + CASE p.value_flag + WHEN 'value_suspect' THEN NULL + WHEN 'annex_suspect' THEN COALESCE(p.signing_value, p.current_value) + ELSE COALESCE(p.current_value, p.signing_value) + END AS trusted_native, + CASE p.value_flag + WHEN 'value_suspect' THEN NULL + WHEN 'annex_suspect' THEN CASE + WHEN p.signing_value IS NOT NULL THEN COALESCE(NULLIF(p.currency, ''), 'BGN') + ELSE p.derived_current_currency + END + ELSE CASE + WHEN p.current_value IS NOT NULL THEN p.derived_current_currency + ELSE COALESCE(NULLIF(p.currency, ''), 'BGN') + END + END AS trusted_currency + FROM paired p +), repaired AS ( + SELECT + id, + derived_current_currency, + CASE + -- value_suspect is repaired from the procedure estimate upstream; do not replace that repair. + WHEN value_flag = 'value_suspect' THEN amount_eur + WHEN trusted_native IS NULL THEN NULL + WHEN trusted_currency = 'EUR' THEN trusted_native + WHEN trusted_currency = 'BGN' THEN trusted_native / 1.95583 + WHEN fx_rate IS NOT NULL THEN trusted_native * fx_rate + ELSE NULL + END AS repaired_amount_eur, + CASE + WHEN value_flag IN ('value_suspect', 'annex_suspect') OR current_value IS NULL THEN NULL + WHEN derived_current_currency = 'EUR' THEN current_value + WHEN derived_current_currency = 'BGN' THEN current_value / 1.95583 + WHEN fx_rate IS NOT NULL THEN current_value * fx_rate + ELSE NULL + END AS repaired_current_value_eur + FROM selected +) +UPDATE contracts +SET + current_value_currency = repaired.derived_current_currency, + amount_eur = repaired.repaired_amount_eur, + current_value_eur = repaired.repaired_current_value_eur +FROM repaired +WHERE repaired.id = contracts.id; + +SELECT + (SELECT COUNT(*) FROM contracts WHERE current_value_currency IS NOT NULL) AS currency_rows, + (SELECT COUNT(*) FROM contracts + WHERE value_flag = 'ok' AND current_value IS NOT NULL + AND (amount_eur IS NULL OR current_value_eur IS NULL + OR ABS(amount_eur - current_value_eur) > 0.01)) AS parity_mismatches; diff --git a/scripts/import.mjs b/scripts/import.mjs index 8b66be22..578a1132 100644 --- a/scripts/import.mjs +++ b/scripts/import.mjs @@ -3,7 +3,15 @@ // both route through scripts/load-eop.mjs; only the date window and derive mode differ. import { execFileSync } from 'node:child_process'; -import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + writeFileSync, +} from 'node:fs'; import { basename, dirname, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { computeCatchupWindow, daysInWindow } from '../packages/ingest/src/ocds.ts'; @@ -272,7 +280,11 @@ async function runWorkBackfill() { if (existsSync(workDb)) rmSync(workDb, { force: true }); console.log(`==> Sigma import (work DB ${workDb})`); - sqliteFile(workDb, resolve(root, 'packages/db/migrations/0000_init.sql')); + const migrationsDir = resolve(root, 'packages/db/migrations'); + const migrations = readdirSync(migrationsDir) + .filter((name) => /^\d+.*\.sql$/.test(name)) + .sort(); + for (const migration of migrations) sqliteFile(workDb, resolve(migrationsDir, migration)); sqliteFile(workDb, resolve(root, 'scripts/work-staging-schema.sql')); let loadFlags = explicitRangeFlags(); diff --git a/scripts/integrity-checks.d.mts b/scripts/integrity-checks.d.mts index 09dcbc0a..d358d0af 100644 --- a/scripts/integrity-checks.d.mts +++ b/scripts/integrity-checks.d.mts @@ -25,6 +25,7 @@ export interface IntegrityResult { export function checkNonEmptyCorpus(runner: IntegrityRunner): Promise; export function checkRollupReconciliation(runner: IntegrityRunner): Promise; +export function checkCurrentAmountParity(runner: IntegrityRunner): Promise; export function checkNoNegativeValues(runner: IntegrityRunner): Promise; export function checkEikValidity(runner: IntegrityRunner): Promise; export function checkDateSanity(runner: IntegrityRunner): Promise; diff --git a/scripts/integrity-checks.mjs b/scripts/integrity-checks.mjs index 7eba68e1..b48c4912 100644 --- a/scripts/integrity-checks.mjs +++ b/scripts/integrity-checks.mjs @@ -153,7 +153,46 @@ export async function checkRollupReconciliation(runner) { }; } -// 2) No negative values feeding the totals. Two classes, split by who controls the defect: +// 2) A clean amended contract has one canonical EUR value. The detail-page timeline reads +// current_value_eur while every aggregate and search result reads amount_eur, so allowing these two +// columns to disagree makes the served database contradict itself even when each rollup reconciles +// perfectly with amount_eur. One cent is the user-visible precision boundary. +export async function checkCurrentAmountParity(runner) { + const name = 'current-amount-parity'; + // current_value_eur is populated by precompute.sql (served D1 / ship-domain), not by normalize on the + // work DB. Before precompute every current_value_eur is NULL, so this parity is only meaningful once + // the rollups exist — gate on home_totals exactly like rollup-reconciliation. + if ( + !(await tableExists(runner, 'contracts')) || + !(await tableExists(runner, 'home_totals')) || + num(await scalar(runner, 'SELECT COUNT(*) AS n FROM home_totals', 'n')) === 0 + ) + return { + name, + ok: true, + skipped: true, + detail: 'precompute rollups absent (current_value_eur not yet populated)', + }; + const mismatches = num( + await scalar( + runner, + "SELECT COUNT(*) AS n FROM contracts WHERE value_flag = 'ok' AND current_value IS NOT NULL " + + 'AND (amount_eur IS NULL OR current_value_eur IS NULL OR ABS(amount_eur - current_value_eur) > 0.01)', + 'n', + ), + ); + return { + name, + ok: mismatches === 0, + skipped: false, + detail: + mismatches === 0 + ? 'all ok current values agree with canonical amount_eur within €0.01' + : `${mismatches} ok contract(s) have amount_eur != current_value_eur by more than €0.01`, + }; +} + +// 3) No negative values feeding the totals. Two classes, split by who controls the defect: // - value_flag='ok' AND amount_eur<0 → HARD fail. A clean row cannot be negative except via a Sigma // derivation bug (e.g. a sign flip); Sigma owns and can fix it. // - any other flag with amount_eur<0 → WARN. normalize keeps value_low rows (set on @@ -215,7 +254,7 @@ export async function checkNoNegativeValues(runner) { }; } -// 3) EIK validity (canonical home: bidders). eik_valid=1 ⇒ eik_normalized is a numeric 9/13-digit +// 4) EIK validity (canonical home: bidders). eik_valid=1 ⇒ eik_normalized is a numeric 9/13-digit // ЕИК; eik_valid<>1 ⇒ eik_normalized IS NULL. normalize-raw guarantees this (it sets // eik_normalized only when eik_valid=1); the gate proves the guarantee held. export async function checkEikValidity(runner) { @@ -252,7 +291,7 @@ export async function checkEikValidity(runner) { }; } -// 4) Date sanity — REPORTED, NOT GATED. Unlike the other checks, signed_at is a pass-through of the +// 5) Date sanity — REPORTED, NOT GATED. Unlike the other checks, signed_at is a pass-through of the // upstream EOP value, not something Sigma derives. An out-of-range date is an upstream record-level // defect (#19–27) that Sigma consumes and cannot correct, so it must NEVER break the daily import — // a single source typo (real example: signed_at='2029-05-14' in the 2024 feed) would otherwise fail @@ -282,7 +321,7 @@ export async function checkDateSanity(runner) { }; } -// 5) Staging → domain reconciliation. normalize-raw records, in one row of pipeline_stats, the +// 6) Staging → domain reconciliation. normalize-raw records, in one row of pipeline_stats, the // eligible-candidate count (the SAME expression the summary prints) and the resulting contracts // count. The gate asserts no contract appeared without an eligible candidate (inserted ≤ // candidates — corroborates the orphan check from the staging side) and that a non-empty corpus @@ -336,6 +375,7 @@ export async function checkStagingReconciliation(runner) { export const CHECKS = [ checkNonEmptyCorpus, checkRollupReconciliation, + checkCurrentAmountParity, checkNoNegativeValues, checkEikValidity, checkDateSanity, diff --git a/scripts/normalize-raw.sql b/scripts/normalize-raw.sql index 6754cb68..ebfef975 100644 --- a/scripts/normalize-raw.sql +++ b/scripts/normalize-raw.sql @@ -122,9 +122,11 @@ SELECT s.authority_eik, act.canonical_type FROM ( - -- Composite joint-procurement EIKs ('EIK1; EIK2') must not mint standalone authorities — - -- they are attributed via their individual members; an unguarded source mints orphan - -- 'auth:EIK1; EIK2' rows referenced by nothing (verified on a full 2020-2026 rebuild). + -- Skip composite joint-procurement EIKs (`EIK1; EIK2`): they must not mint a standalone + -- `auth:EIK1; EIK2` authority. Joint tenders are attributed to a single lead (joint_tender_leads) + -- and split to individual co-authorities (contract_co_authorities); the composite is referenced by + -- nothing, so it would only be an orphan row. Individual members still enter via their standalone + -- occurrences and member_defaults. SELECT authority_eik FROM raw_contracts WHERE authority_eik IS NOT NULL AND authority_eik NOT LIKE '%;%' UNION SELECT authority_eik FROM raw_tenders WHERE authority_eik IS NOT NULL AND authority_eik NOT LIKE '%;%' @@ -210,8 +212,10 @@ INSERT OR IGNORE INTO joint_authority_members (unp, authority_id, member_name, authority_type, source_ordinal) SELECT unp, 'auth:' || authority_eik, NULLIF(member_name, ''), authority_type, source_ordinal FROM split --- A member EIK still carrying ';' is a composite the split could not decompose - not a real --- single authority; it must not seed a member row or mint an orphan 'auth:EIK1; EIK2'. +-- A member whose EIK still carries a ';' is a composite that the split could not decompose (unusual +-- delimiter/format). It is not a real single authority, so it must never seed a member row — otherwise +-- member_defaults mints an orphan 'auth:EIK1; EIK2' authority that nothing references. The lead is still +-- attributed via joint_tender_leads; only the (already-unusable) co-authority breakdown is skipped. WHERE authority_eik <> '' AND authority_eik NOT LIKE '%;%'; -- A minority of co-authorities never occur standalone. Mint those real EIK identities from the @@ -709,13 +713,44 @@ SET ownership_kind = ( -- the rate, and the EUR value are all auditable without joining fx_rates. INSERT OR IGNORE INTO contracts (id, tender_id, bidder_id, ordering_unit_name, amount, currency, signed_at, - contract_number, signing_value, current_value, annex_count, eu_funded, bids_received, + contract_number, signing_value, current_value, current_value_currency, annex_count, eu_funded, bids_received, contract_kind, awarded_to_group, value_flag, date_flag, amount_eur, fx_converted, fx_rate, lot_id, document_number, published_at, contract_subject, eu_programme, duration_days, winner_size, contractor_country, bids_sme, bids_rejected, bids_non_eea, subcontractor_eik, subcontractor_name, subcontract_value, eauction, framework, accelerated, strategic) +-- amendment_winner: the currency of whichever raw_amendments row supplied contract_number's +-- current_value (derive-amendments.sql's own rollup, mirrored here so the winning currency +-- travels alongside the value it minted — computed ONCE over raw_amendments, not per-row). +WITH amendment_dedup AS ( + SELECT *, + 'am:' || COALESCE(unp, '') || ':' || COALESCE(contract_number, '') || ':' || + COALESCE( + NULLIF(document_number, ''), + NULLIF(correction_number, ''), + NULLIF(seq_no, ''), + 'content:' || COALESCE(published_at, '') || ':' || + COALESCE(CAST(value_before AS TEXT), '') || ':' || + COALESCE(CAST(value_after AS TEXT), '') || ':' || + COALESCE(CAST(value_delta AS TEXT), '') || ':' || + COALESCE(currency, '') || ':' || + COALESCE(description, '') + ) AS natural_key + FROM raw_amendments +), amendment_rn AS ( + SELECT *, + ROW_NUMBER() OVER (PARTITION BY natural_key ORDER BY source DESC, id DESC) AS rn + FROM amendment_dedup +), amendment_winner AS ( + SELECT unp, contract_number, currency, + ROW_NUMBER() OVER ( + PARTITION BY unp, contract_number + ORDER BY published_at DESC, natural_key DESC + ) AS win_rn + FROM amendment_rn + WHERE rn = 1 AND value_after IS NOT NULL +) SELECT CASE WHEN x.source LIKE 'eop:%' THEN 'c:e:' || COALESCE(x.unp, '') || ':' || COALESCE(x.contract_number, '') || ':' || @@ -733,6 +768,7 @@ SELECT x.contract_number, x.signing_value, x.current_value, + x.current_value_currency, COALESCE(x.annex_count, 0), x.eu_funded, x.bids_received, @@ -743,8 +779,8 @@ SELECT CASE WHEN x.value_flag = 'value_suspect' THEN x.proc_est_eur WHEN x.trusted_native IS NULL THEN NULL - WHEN COALESCE(x.currency, 'BGN') = 'EUR' THEN x.trusted_native - WHEN COALESCE(x.currency, 'BGN') = 'BGN' THEN x.trusted_native / 1.95583 + WHEN x.trusted_currency = 'EUR' THEN x.trusted_native + WHEN x.trusted_currency = 'BGN' THEN x.trusted_native / 1.95583 ELSE x.trusted_native * x.fx_rate END, CASE WHEN COALESCE(x.currency, 'BGN') NOT IN ('BGN', 'EUR') THEN 1 ELSE 0 END, @@ -785,6 +821,27 @@ FROM ( WHEN 'annex_suspect' THEN COALESCE(y.signing_value, y.current_value) ELSE COALESCE(y.current_value, y.signing_value) END AS trusted_native, + -- Keep the companion currency paired with the exact native value chosen above. In particular, + -- annex_suspect normally falls back to signing_value, which remains in contracts.currency even + -- when the rejected amendment used a different currency. + CASE y.value_flag + WHEN 'value_suspect' THEN NULL + WHEN 'annex_suspect' THEN CASE + WHEN y.signing_value IS NOT NULL THEN COALESCE(NULLIF(y.currency, ''), 'BGN') + ELSE COALESCE(NULLIF(y.amendment_currency, ''), NULLIF(y.currency, ''), 'BGN') + END + ELSE CASE + WHEN y.current_value IS NOT NULL THEN COALESCE(NULLIF(y.amendment_currency, ''), NULLIF(y.currency, ''), 'BGN') + ELSE COALESCE(NULLIF(y.currency, ''), 'BGN') + END + END AS trusted_currency, + -- current_value can be denominated in a DIFFERENT currency than the contract's own (when the + -- winning amendment_winner row recorded one) — precompute.sql's later current_value_eur pass + -- uses this column instead of `currency` so that amendment isn't re-converted a second time. + CASE WHEN y.current_value IS NOT NULL + THEN COALESCE(NULLIF(y.amendment_currency, ''), NULLIF(y.currency, ''), 'BGN') + ELSE COALESCE(NULLIF(y.currency, ''), 'BGN') + END AS current_value_currency, -- ECB rates are published on business days only; carry the latest prior rate forward for -- weekend/holiday signings, never future-dated, and cap the fallback at 10 calendar days. CASE WHEN COALESCE(y.currency, 'BGN') NOT IN ('BGN', 'EUR') @@ -880,10 +937,22 @@ FROM ( FROM ( SELECT c.*, ci.bidder_key, c.authority_name AS authority_name_raw, - CASE - WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN COALESCE(c.current_value, c.signing_value) - WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'BGN' THEN COALESCE(c.current_value, c.signing_value) / 1.95583 - ELSE COALESCE(c.current_value, c.signing_value) * ( + CASE WHEN c.current_value IS NOT NULL THEN CASE + WHEN COALESCE(NULLIF(aw.currency, ''), NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN c.current_value + WHEN COALESCE(NULLIF(aw.currency, ''), NULLIF(c.currency, ''), 'BGN') = 'BGN' THEN c.current_value / 1.95583 + ELSE c.current_value * ( + SELECT f.eur_per_unit + FROM fx_rates f + WHERE f.base_currency = COALESCE(NULLIF(aw.currency, ''), NULLIF(c.currency, '')) + AND f.rate_date <= c.contract_date + AND f.rate_date >= date(c.contract_date, '-10 days') + ORDER BY f.rate_date DESC + LIMIT 1 + ) + END ELSE CASE + WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN c.signing_value + WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'BGN' THEN c.signing_value / 1.95583 + ELSE c.signing_value * ( SELECT f.eur_per_unit FROM fx_rates f WHERE f.base_currency = NULLIF(c.currency, '') @@ -892,7 +961,7 @@ FROM ( ORDER BY f.rate_date DESC LIMIT 1 ) - END AS eff_eur, + END END AS eff_eur, CASE WHEN t.estimated_value IS NULL THEN NULL WHEN COALESCE(NULLIF(t.currency, ''), 'BGN') = 'EUR' THEN t.estimated_value @@ -907,11 +976,14 @@ FROM ( LIMIT 1 ) END AS proc_est_eur, - t.estimated_value AS proc_est_native + t.estimated_value AS proc_est_native, + aw.currency AS amendment_currency FROM raw_contracts c JOIN contractor_identity ci ON c.contractor_eik IS ci.eik_raw AND c.contractor_name IS ci.name_raw LEFT JOIN tenders t ON t.id = 't:' || c.unp + LEFT JOIN amendment_winner aw + ON aw.unp = c.unp AND aw.contract_number = c.contract_number AND aw.win_rn = 1 ) c -- EOP always; an OCDS row only when no EOP row shares its contract_number - EOP wins. -- Key is contract_number (the public-procurement contract document number, common to both feeds), NOT unp: @@ -1099,6 +1171,33 @@ CREATE TABLE IF NOT EXISTS pipeline_stats ( computed_at TEXT NOT NULL ); DELETE FROM pipeline_stats; +WITH amendment_dedup AS ( + SELECT *, + 'am:' || COALESCE(unp, '') || ':' || COALESCE(contract_number, '') || ':' || + COALESCE( + NULLIF(document_number, ''), + NULLIF(correction_number, ''), + NULLIF(seq_no, ''), + 'content:' || COALESCE(published_at, '') || ':' || + COALESCE(CAST(value_before AS TEXT), '') || ':' || + COALESCE(CAST(value_after AS TEXT), '') || ':' || + COALESCE(CAST(value_delta AS TEXT), '') || ':' || + COALESCE(currency, '') || ':' || + COALESCE(description, '') + ) AS natural_key + FROM raw_amendments +), amendment_rn AS ( + SELECT *, ROW_NUMBER() OVER (PARTITION BY natural_key ORDER BY source DESC, id DESC) AS rn + FROM amendment_dedup +), amendment_winner AS ( + SELECT unp, contract_number, currency, + ROW_NUMBER() OVER ( + PARTITION BY unp, contract_number + ORDER BY published_at DESC, natural_key DESC + ) AS win_rn + FROM amendment_rn + WHERE rn = 1 AND value_after IS NOT NULL +) INSERT INTO pipeline_stats (id, contract_candidates, contracts_inserted, computed_at) SELECT 1, (SELECT COUNT(*) FROM ( @@ -1159,10 +1258,22 @@ SELECT 1, c.bidder_key FROM ( SELECT c.*, ci.bidder_key, - CASE - WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN COALESCE(c.current_value, c.signing_value) - WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'BGN' THEN COALESCE(c.current_value, c.signing_value) / 1.95583 - ELSE COALESCE(c.current_value, c.signing_value) * ( + CASE WHEN c.current_value IS NOT NULL THEN CASE + WHEN COALESCE(NULLIF(aw.currency, ''), NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN c.current_value + WHEN COALESCE(NULLIF(aw.currency, ''), NULLIF(c.currency, ''), 'BGN') = 'BGN' THEN c.current_value / 1.95583 + ELSE c.current_value * ( + SELECT f.eur_per_unit + FROM fx_rates f + WHERE f.base_currency = COALESCE(NULLIF(aw.currency, ''), NULLIF(c.currency, '')) + AND f.rate_date <= c.contract_date + AND f.rate_date >= date(c.contract_date, '-10 days') + ORDER BY f.rate_date DESC + LIMIT 1 + ) + END ELSE CASE + WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN c.signing_value + WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'BGN' THEN c.signing_value / 1.95583 + ELSE c.signing_value * ( SELECT f.eur_per_unit FROM fx_rates f WHERE f.base_currency = NULLIF(c.currency, '') @@ -1171,7 +1282,7 @@ SELECT 1, ORDER BY f.rate_date DESC LIMIT 1 ) - END AS eff_eur, + END END AS eff_eur, CASE WHEN t.estimated_value IS NULL THEN NULL WHEN COALESCE(NULLIF(t.currency, ''), 'BGN') = 'EUR' THEN t.estimated_value @@ -1186,11 +1297,14 @@ SELECT 1, LIMIT 1 ) END AS proc_est_eur, - t.estimated_value AS proc_est_native + t.estimated_value AS proc_est_native, + aw.currency AS amendment_currency FROM raw_contracts c JOIN contractor_identity ci ON c.contractor_eik IS ci.eik_raw AND c.contractor_name IS ci.name_raw LEFT JOIN tenders t ON t.id = 't:' || c.unp + LEFT JOIN amendment_winner aw + ON aw.unp = c.unp AND aw.contract_number = c.contract_number AND aw.win_rn = 1 ) c -- Eligibility must mirror the INSERT INTO contracts WHERE exactly, so this candidate count is a -- true superset of what lands (inserted <= candidates holds by construction; the gap is only the diff --git a/scripts/precompute.sql b/scripts/precompute.sql index bdecce18..914b3aa7 100644 --- a/scripts/precompute.sql +++ b/scripts/precompute.sql @@ -34,8 +34,8 @@ UPDATE contracts SET ELSE NULL END, current_value_eur = CASE WHEN value_flag IN ('value_suspect','annex_suspect') OR current_value IS NULL THEN NULL - WHEN COALESCE(currency,'BGN') = 'EUR' THEN current_value - WHEN COALESCE(currency,'BGN') = 'BGN' THEN current_value / 1.95583 + WHEN COALESCE(NULLIF(current_value_currency, ''), NULLIF(currency, ''), 'BGN') = 'EUR' THEN current_value + WHEN COALESCE(NULLIF(current_value_currency, ''), NULLIF(currency, ''), 'BGN') = 'BGN' THEN current_value / 1.95583 WHEN fx_rate IS NOT NULL THEN current_value * fx_rate ELSE NULL END; diff --git a/scripts/refresh-slice.sql b/scripts/refresh-slice.sql index 8b7a5f67..ff333431 100644 --- a/scripts/refresh-slice.sql +++ b/scripts/refresh-slice.sql @@ -20,9 +20,37 @@ DROP TABLE IF EXISTS refresh_joint_tender_leads; DROP TABLE IF EXISTS refresh_unp_prefix_authorities; DROP TABLE IF EXISTS refresh_joint_authority_members; DROP TABLE IF EXISTS refresh_joint_tender_sources; +DROP TABLE IF EXISTS refresh_amendment_winners; CREATE TABLE refresh_touched_contracts (id TEXT PRIMARY KEY); CREATE TABLE refresh_touched_bidders (bidder_id TEXT PRIMARY KEY); CREATE TABLE refresh_touched_authorities (authority_id TEXT PRIMARY KEY); +CREATE TABLE refresh_amendment_winners AS +WITH keyed AS ( + SELECT *, + 'am:' || COALESCE(unp, '') || ':' || COALESCE(contract_number, '') || ':' || + COALESCE( + NULLIF(document_number, ''), NULLIF(correction_number, ''), NULLIF(seq_no, ''), + 'content:' || COALESCE(published_at, '') || ':' || + COALESCE(CAST(value_before AS TEXT), '') || ':' || + COALESCE(CAST(value_after AS TEXT), '') || ':' || + COALESCE(CAST(value_delta AS TEXT), '') || ':' || + COALESCE(currency, '') || ':' || COALESCE(description, '') + ) AS natural_key + FROM raw_amendments +), dedup AS ( + SELECT *, ROW_NUMBER() OVER (PARTITION BY natural_key ORDER BY source DESC, id DESC) AS rn + FROM keyed +), winners AS ( + SELECT unp, contract_number, currency, + ROW_NUMBER() OVER ( + PARTITION BY unp, contract_number ORDER BY published_at DESC, natural_key DESC + ) AS win_rn + FROM dedup + WHERE rn = 1 AND value_after IS NOT NULL +) +SELECT unp, contract_number, currency FROM winners WHERE win_rn = 1; +CREATE INDEX idx_refresh_amendment_winners + ON refresh_amendment_winners(unp, contract_number); -- @refresh-batch authorities-bidders -- ── 1) Authorities referenced by OCDS staging (new ones only; INSERT OR IGNORE) ──────────────────── @@ -182,12 +210,12 @@ WITH RECURSIVE split ( FROM split WHERE TRIM(eik_rest) <> '' ) +-- Guard composite EIKs the split could not decompose (see normalize-raw.sql): a ';'-bearing member +-- is not a real single authority and must not seed a member row or an orphan 'auth:EIK1; EIK2'. INSERT OR IGNORE INTO refresh_joint_authority_members (unp, authority_id, member_name, authority_type, source_ordinal) SELECT unp, 'auth:' || authority_eik, NULLIF(member_name, ''), authority_type, source_ordinal FROM split --- A member EIK still carrying ';' is a composite the split could not decompose - not a real --- single authority; it must not seed a member row or mint an orphan 'auth:EIK1; EIK2'. WHERE authority_eik <> '' AND authority_eik NOT LIKE '%;%'; WITH name_counts AS ( @@ -904,6 +932,7 @@ WHERE id IN ( ); INSERT OR IGNORE INTO contracts (id, tender_id, bidder_id, ordering_unit_name, amount, currency, signed_at, contract_number, signing_value, current_value, + current_value_currency, annex_count, eu_funded, bids_received, contract_kind, awarded_to_group, value_flag, date_flag, amount_eur, fx_converted, fx_rate, signing_value_eur, current_value_eur, lot_id, document_number, published_at, contract_subject, @@ -923,6 +952,7 @@ SELECT x.contract_number, x.signing_value, x.current_value, + x.current_value_currency, 0, x.eu_funded, x.bids_received, @@ -959,8 +989,8 @@ FROM ( -- so it counts in every sum; it is merely labelled in the UI. annex_suspect uses trusted_native's signing fallback. CASE WHEN q.value_flag = 'value_suspect' THEN q.proc_est_eur - WHEN COALESCE(q.currency,'BGN') = 'EUR' THEN q.trusted_native - WHEN COALESCE(q.currency,'BGN') = 'BGN' THEN q.trusted_native / 1.95583 + WHEN q.trusted_currency = 'EUR' THEN q.trusted_native + WHEN q.trusted_currency = 'BGN' THEN q.trusted_native / 1.95583 ELSE q.trusted_native * q.fx_rate END AS amount_eur, CASE @@ -971,8 +1001,8 @@ FROM ( END AS signing_value_eur, CASE WHEN q.value_flag IN ('value_suspect', 'annex_suspect') OR q.current_value IS NULL THEN NULL - WHEN COALESCE(q.currency,'BGN') = 'EUR' THEN q.current_value - WHEN COALESCE(q.currency,'BGN') = 'BGN' THEN q.current_value / 1.95583 + WHEN q.current_value_currency = 'EUR' THEN q.current_value + WHEN q.current_value_currency = 'BGN' THEN q.current_value / 1.95583 ELSE q.current_value * q.fx_rate END AS current_value_eur FROM ( @@ -987,6 +1017,24 @@ FROM ( WHEN 'annex_suspect' THEN COALESCE(y.signing_value, y.current_value) ELSE COALESCE(y.current_value, y.signing_value) END AS trusted_native, + CASE y.value_flag + WHEN 'value_suspect' THEN NULL + WHEN 'annex_suspect' THEN CASE + WHEN y.signing_value IS NOT NULL THEN COALESCE(NULLIF(y.currency, ''), 'BGN') + ELSE COALESCE((SELECT NULLIF(w.currency, '') FROM refresh_amendment_winners w + WHERE w.unp = y.unp AND w.contract_number = y.contract_number), NULLIF(y.currency, ''), 'BGN') + END + ELSE CASE + WHEN y.current_value IS NOT NULL THEN COALESCE((SELECT NULLIF(w.currency, '') FROM refresh_amendment_winners w + WHERE w.unp = y.unp AND w.contract_number = y.contract_number), NULLIF(y.currency, ''), 'BGN') + ELSE COALESCE(NULLIF(y.currency, ''), 'BGN') + END + END AS trusted_currency, + CASE WHEN y.current_value IS NOT NULL + THEN COALESCE((SELECT NULLIF(w.currency, '') FROM refresh_amendment_winners w + WHERE w.unp = y.unp AND w.contract_number = y.contract_number), NULLIF(y.currency, ''), 'BGN') + ELSE COALESCE(NULLIF(y.currency, ''), 'BGN') + END AS current_value_currency, -- fx: EUR as-is, BGN at the peg, foreign at the signing-date ECB rate (NULL if missing) CASE WHEN COALESCE(y.currency,'BGN') NOT IN ('BGN','EUR') THEN ( @@ -1072,10 +1120,25 @@ FROM ( FROM ( SELECT c.*, ci.bidder_key, c.authority_name AS authority_name_raw, - CASE - WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN COALESCE(c.current_value, c.signing_value) - WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'BGN' THEN COALESCE(c.current_value, c.signing_value) / 1.95583 - ELSE COALESCE(c.current_value, c.signing_value) * ( + CASE WHEN c.current_value IS NOT NULL THEN CASE + WHEN COALESCE((SELECT NULLIF(w.currency, '') FROM refresh_amendment_winners w + WHERE w.unp = c.unp AND w.contract_number = c.contract_number), NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN c.current_value + WHEN COALESCE((SELECT NULLIF(w.currency, '') FROM refresh_amendment_winners w + WHERE w.unp = c.unp AND w.contract_number = c.contract_number), NULLIF(c.currency, ''), 'BGN') = 'BGN' THEN c.current_value / 1.95583 + ELSE c.current_value * ( + SELECT f.eur_per_unit + FROM fx_rates f + WHERE f.base_currency = COALESCE((SELECT NULLIF(w.currency, '') FROM refresh_amendment_winners w + WHERE w.unp = c.unp AND w.contract_number = c.contract_number), NULLIF(c.currency, '')) + AND f.rate_date <= c.contract_date + AND f.rate_date >= date(c.contract_date, '-10 days') + ORDER BY f.rate_date DESC + LIMIT 1 + ) + END ELSE CASE + WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN c.signing_value + WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'BGN' THEN c.signing_value / 1.95583 + ELSE c.signing_value * ( SELECT f.eur_per_unit FROM fx_rates f WHERE f.base_currency = NULLIF(c.currency, '') @@ -1084,7 +1147,7 @@ FROM ( ORDER BY f.rate_date DESC LIMIT 1 ) - END AS eff_eur, + END END AS eff_eur, CASE WHEN t.estimated_value IS NULL THEN NULL WHEN COALESCE(NULLIF(t.currency, ''), 'BGN') = 'EUR' THEN t.estimated_value @@ -1145,6 +1208,7 @@ WHERE id IN ( INSERT OR IGNORE INTO contracts (id, tender_id, bidder_id, ordering_unit_name, amount, currency, signed_at, contract_number, signing_value, current_value, + current_value_currency, annex_count, eu_funded, bids_received, contract_kind, awarded_to_group, value_flag, date_flag, amount_eur, fx_converted, fx_rate, signing_value_eur, current_value_eur, lot_id, document_number, published_at, contract_subject, @@ -1164,6 +1228,7 @@ SELECT x.contract_number, x.signing_value, x.current_value, + x.current_value_currency, COALESCE(x.annex_count, 0), x.eu_funded, x.bids_received, @@ -1200,8 +1265,8 @@ FROM ( -- so it counts in every sum; it is merely labelled in the UI. annex_suspect uses trusted_native's signing fallback. CASE WHEN q.value_flag = 'value_suspect' THEN q.proc_est_eur - WHEN COALESCE(q.currency,'BGN') = 'EUR' THEN q.trusted_native - WHEN COALESCE(q.currency,'BGN') = 'BGN' THEN q.trusted_native / 1.95583 + WHEN q.trusted_currency = 'EUR' THEN q.trusted_native + WHEN q.trusted_currency = 'BGN' THEN q.trusted_native / 1.95583 ELSE q.trusted_native * q.fx_rate END AS amount_eur, CASE @@ -1212,8 +1277,8 @@ FROM ( END AS signing_value_eur, CASE WHEN q.value_flag IN ('value_suspect', 'annex_suspect') OR q.current_value IS NULL THEN NULL - WHEN COALESCE(q.currency,'BGN') = 'EUR' THEN q.current_value - WHEN COALESCE(q.currency,'BGN') = 'BGN' THEN q.current_value / 1.95583 + WHEN q.current_value_currency = 'EUR' THEN q.current_value + WHEN q.current_value_currency = 'BGN' THEN q.current_value / 1.95583 ELSE q.current_value * q.fx_rate END AS current_value_eur FROM ( @@ -1228,6 +1293,24 @@ FROM ( WHEN 'annex_suspect' THEN COALESCE(y.signing_value, y.current_value) ELSE COALESCE(y.current_value, y.signing_value) END AS trusted_native, + CASE y.value_flag + WHEN 'value_suspect' THEN NULL + WHEN 'annex_suspect' THEN CASE + WHEN y.signing_value IS NOT NULL THEN COALESCE(NULLIF(y.currency, ''), 'BGN') + ELSE COALESCE((SELECT NULLIF(w.currency, '') FROM refresh_amendment_winners w + WHERE w.unp = y.unp AND w.contract_number = y.contract_number), NULLIF(y.currency, ''), 'BGN') + END + ELSE CASE + WHEN y.current_value IS NOT NULL THEN COALESCE((SELECT NULLIF(w.currency, '') FROM refresh_amendment_winners w + WHERE w.unp = y.unp AND w.contract_number = y.contract_number), NULLIF(y.currency, ''), 'BGN') + ELSE COALESCE(NULLIF(y.currency, ''), 'BGN') + END + END AS trusted_currency, + CASE WHEN y.current_value IS NOT NULL + THEN COALESCE((SELECT NULLIF(w.currency, '') FROM refresh_amendment_winners w + WHERE w.unp = y.unp AND w.contract_number = y.contract_number), NULLIF(y.currency, ''), 'BGN') + ELSE COALESCE(NULLIF(y.currency, ''), 'BGN') + END AS current_value_currency, CASE WHEN COALESCE(y.currency,'BGN') NOT IN ('BGN','EUR') THEN ( SELECT f.eur_per_unit @@ -1317,10 +1400,25 @@ FROM ( FROM ( SELECT c.*, ci.bidder_key, c.authority_name AS authority_name_raw, - CASE - WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN COALESCE(c.current_value, c.signing_value) - WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'BGN' THEN COALESCE(c.current_value, c.signing_value) / 1.95583 - ELSE COALESCE(c.current_value, c.signing_value) * ( + CASE WHEN c.current_value IS NOT NULL THEN CASE + WHEN COALESCE((SELECT NULLIF(w.currency, '') FROM refresh_amendment_winners w + WHERE w.unp = c.unp AND w.contract_number = c.contract_number), NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN c.current_value + WHEN COALESCE((SELECT NULLIF(w.currency, '') FROM refresh_amendment_winners w + WHERE w.unp = c.unp AND w.contract_number = c.contract_number), NULLIF(c.currency, ''), 'BGN') = 'BGN' THEN c.current_value / 1.95583 + ELSE c.current_value * ( + SELECT f.eur_per_unit + FROM fx_rates f + WHERE f.base_currency = COALESCE((SELECT NULLIF(w.currency, '') FROM refresh_amendment_winners w + WHERE w.unp = c.unp AND w.contract_number = c.contract_number), NULLIF(c.currency, '')) + AND f.rate_date <= c.contract_date + AND f.rate_date >= date(c.contract_date, '-10 days') + ORDER BY f.rate_date DESC + LIMIT 1 + ) + END ELSE CASE + WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN c.signing_value + WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'BGN' THEN c.signing_value / 1.95583 + ELSE c.signing_value * ( SELECT f.eur_per_unit FROM fx_rates f WHERE f.base_currency = NULLIF(c.currency, '') @@ -1329,7 +1427,7 @@ FROM ( ORDER BY f.rate_date DESC LIMIT 1 ) - END AS eff_eur, + END END AS eff_eur, CASE WHEN t.estimated_value IS NULL THEN NULL WHEN COALESCE(NULLIF(t.currency, ''), 'BGN') = 'EUR' THEN t.estimated_value @@ -1463,6 +1561,14 @@ SET AND a.value_after IS NOT NULL ORDER BY a.published_at DESC, a.id DESC LIMIT 1 + ), + current_value_currency = ( + SELECT COALESCE(NULLIF(a.currency, ''), contracts.currency) FROM amendments a + WHERE a.unp = substr(contracts.tender_id, 3) + AND a.contract_number = contracts.contract_number + AND a.value_after IS NOT NULL + ORDER BY a.published_at DESC, a.id DESC + LIMIT 1 ) WHERE (id GLOB 'c:[eo]:*' AND EXISTS ( SELECT 1 FROM raw_contracts rc @@ -1476,12 +1582,22 @@ WHERE (id GLOB 'c:[eo]:*' AND EXISTS ( ); WITH contract_base AS ( - SELECT c.id, c.currency, c.signing_value, c.current_value, c.fx_rate, c.value_flag, + SELECT c.id, c.currency, c.signing_value, c.current_value, c.current_value_currency, c.fx_rate, c.value_flag, te.estimated_value AS proc_est_native, CASE - WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN COALESCE(c.current_value, c.signing_value) - WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'BGN' THEN COALESCE(c.current_value, c.signing_value) / 1.95583 - WHEN c.fx_rate IS NOT NULL THEN COALESCE(c.current_value, c.signing_value) * c.fx_rate + -- current_value (when present) is denominated in current_value_currency (whichever + -- amendment last set it), NOT contracts.currency — that's the contract's original + -- signing currency and only applies to the signing_value fallback. + WHEN c.current_value IS NOT NULL THEN + CASE + WHEN COALESCE(NULLIF(c.current_value_currency, ''), NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN c.current_value + WHEN COALESCE(NULLIF(c.current_value_currency, ''), NULLIF(c.currency, ''), 'BGN') = 'BGN' THEN c.current_value / 1.95583 + WHEN c.fx_rate IS NOT NULL THEN c.current_value * c.fx_rate + ELSE NULL + END + WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN c.signing_value + WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'BGN' THEN c.signing_value / 1.95583 + WHEN c.fx_rate IS NOT NULL THEN c.signing_value * c.fx_rate ELSE NULL END AS eff_eur, CASE @@ -1531,7 +1647,7 @@ WITH contract_base AS ( AND a.contract_number = c.contract_number ) ), base AS ( - SELECT id, currency, signing_value, current_value, fx_rate, proc_est_eur, proc_est_native, + SELECT id, currency, signing_value, current_value, current_value_currency, fx_rate, proc_est_eur, proc_est_native, CASE WHEN c.value_flag <> 'annex_suspect' AND NOT (c.current_value IS NOT NULL AND (c.current_value < 0 OR (c.signing_value > 0 AND c.current_value / c.signing_value >= 100))) @@ -1554,10 +1670,21 @@ WITH contract_base AS ( WHEN 'annex_suspect' THEN COALESCE(signing_value, current_value) ELSE COALESCE(current_value, signing_value) END AS trusted_native, + CASE new_value_flag + WHEN 'value_suspect' THEN NULL + WHEN 'annex_suspect' THEN CASE + WHEN signing_value IS NOT NULL THEN COALESCE(NULLIF(currency, ''), 'BGN') + ELSE COALESCE(NULLIF(current_value_currency, ''), NULLIF(currency, ''), 'BGN') + END + ELSE CASE + WHEN current_value IS NOT NULL THEN COALESCE(NULLIF(current_value_currency, ''), NULLIF(currency, ''), 'BGN') + ELSE COALESCE(NULLIF(currency, ''), 'BGN') + END + END AS trusted_currency, CASE WHEN new_value_flag IN ('value_suspect', 'annex_suspect') OR current_value IS NULL THEN NULL - WHEN COALESCE(currency, 'BGN') = 'EUR' THEN current_value - WHEN COALESCE(currency, 'BGN') = 'BGN' THEN current_value / 1.95583 + WHEN COALESCE(NULLIF(current_value_currency, ''), NULLIF(currency, ''), 'BGN') = 'EUR' THEN current_value + WHEN COALESCE(NULLIF(current_value_currency, ''), NULLIF(currency, ''), 'BGN') = 'BGN' THEN current_value / 1.95583 WHEN fx_rate IS NOT NULL THEN current_value * fx_rate ELSE NULL END AS new_current_value_eur, @@ -1570,8 +1697,8 @@ WITH contract_base AS ( CASE WHEN new_value_flag = 'value_suspect' THEN proc_est_eur WHEN trusted_native IS NULL THEN NULL - WHEN COALESCE(currency, 'BGN') = 'EUR' THEN trusted_native - WHEN COALESCE(currency, 'BGN') = 'BGN' THEN trusted_native / 1.95583 + WHEN trusted_currency = 'EUR' THEN trusted_native + WHEN trusted_currency = 'BGN' THEN trusted_native / 1.95583 WHEN fx_rate IS NOT NULL THEN trusted_native * fx_rate ELSE NULL END AS new_amount_eur, @@ -1769,3 +1896,4 @@ DROP TABLE IF EXISTS refresh_joint_tender_sources; DROP TABLE IF EXISTS refresh_touched_contracts; DROP TABLE IF EXISTS refresh_touched_bidders; DROP TABLE IF EXISTS refresh_touched_authorities; +DROP TABLE IF EXISTS refresh_amendment_winners;