diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 5d5ca680..835e208d 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -214,6 +214,65 @@ jobs: ;; esac + # #305 additive columns (migrations 0006/0007) must exist BEFORE the Worker serves: the contract-page + # query reads amendments.value_restated/value_suspect, and promote-amendments/refresh-slice also INSERT + # value_treatment — so all three must be present or the read AND the ETL write fail. Same rationale as + # the currency step above: the migration ledger is empty, so `d1 migrations apply` would collide on + # 0000. Probe the actual table and ALTER only the missing columns (SQLite has no ADD COLUMN IF NOT + # EXISTS). These are pure additions with safe defaults (INTEGER NOT NULL DEFAULT 0 / nullable TEXT), so + # no backfill or completion marker is needed — an ALTER populates every existing row. Malformed + # responses are fatal. This also makes 0006/0007 replay-safe when they were applied out-of-ledger. + - name: Apply amendment restated/suspect columns + if: steps.guard.outputs.ok == 'true' + run: | + node scripts/wrangler-render.mjs apps/web/wrangler.jsonc + schema_json="$(pnpm --filter @sigma/web exec wrangler d1 execute "${SIGMA_D1_NAME:-sigma}" \ + --config wrangler.deploy.jsonc --remote --yes --json \ + --command "SELECT + (SELECT COUNT(*) FROM pragma_table_info('amendments') WHERE name = 'value_restated') AS has_restated, + (SELECT COUNT(*) FROM pragma_table_info('amendments') WHERE name = 'value_treatment') AS has_treatment, + (SELECT COUNT(*) FROM pragma_table_info('amendments') WHERE name = 'value_suspect') AS has_suspect")" + + read_flag() { + printf '%s' "$schema_json" | node -e ' + const fs = require("fs"); + let payload; + try { + payload = JSON.parse(fs.readFileSync(0, "utf8")); + } catch { + process.exit(2); + } + const result = Array.isArray(payload) ? payload[0] : payload; + const row = result && Array.isArray(result.results) ? result.results[0] : null; + const key = process.argv[1]; + if (!row || !Object.hasOwn(row, key)) process.exit(2); + process.exit(Number(row[key]) === 1 ? 0 : 1); + ' "$1" + } + + add_column() { + echo "amendments.$1 missing; adding it." + pnpm --filter @sigma/web exec wrangler d1 execute "${SIGMA_D1_NAME:-sigma}" \ + --config wrangler.deploy.jsonc --remote --yes \ + --command "ALTER TABLE amendments ADD COLUMN $2" + } + + ensure_column() { + set +e + read_flag "$1" + status="$?" + set -e + case "$status" in + 0) echo "amendments.$1 already exists." ;; + 1) add_column "$1" "$2" ;; + *) echo "::error::Could not determine whether amendments.$1 exists."; exit 1 ;; + esac + } + + ensure_column value_restated "value_restated INTEGER NOT NULL DEFAULT 0" + ensure_column value_treatment "value_treatment TEXT" + ensure_column value_suspect "value_suspect INTEGER NOT NULL DEFAULT 0" + # `run deploy`, not `deploy` — bare `pnpm deploy` is a pnpm built-in, not our package script. - name: Deploy explorer (sigma) if: steps.guard.outputs.ok == 'true' diff --git a/apps/web/app/lib/assistant/describe-schema.ts b/apps/web/app/lib/assistant/describe-schema.ts index 6157b382..8e887417 100644 --- a/apps/web/app/lib/assistant/describe-schema.ts +++ b/apps/web/app/lib/assistant/describe-schema.ts @@ -10,11 +10,13 @@ export const DATA_TRAPS: string[] = [ 'Парични агрегати: СУМИРАЙ САМО `contracts.amount_eur` (каноничен EUR, безопасен за сумиране). ' + 'НИКОГА не сумирай `contracts.amount` — то е „както е записано" в смесена валута (`currency`), само за показване.', 'Канонична база за всяка парична сума: `contracts.amount_eur IS NOT NULL`. НЕ филтрирай по ' + - '`value_flag`: включи `ok`, `review`, `annex_suspect`, `value_low` и поправените `value_suspect` редове.', + '`value_flag`: включи `ok`, `review`, `annex_suspect`, `annex_total_suspect`, `value_low` и ' + + 'поправените `value_suspect` редове.', '`amount_eur IS NULL` означава, че няма използваема EUR стойност (например `value_suspect` без ' + 'прогноза за поправка или чужда валута без FX курс); само тези редове се изключват от парични суми.', - '`value_flag` ∈ {ok, review, annex_suspect, value_suspect, value_low} мени значението на стойността ' + - 'на реда, но не и каноничната база; `date_flag` ∈ {ok, signed_after_publication} е вердикт за датата.', + '`value_flag` ∈ {ok, review, annex_suspect, annex_total_suspect, value_suspect, value_low} мени ' + + 'значението на стойността на реда, но не и каноничната база; `date_flag` ∈ {ok, ' + + 'signed_after_publication} е вердикт за датата.', "`tenders.procedure_type = 'неизвестна'` маркира СИНТЕТИЧНИ (само-договорни) преписки — " + 'изключи ги при анализ на разпределението по процедура, освен ако нарочно ги искаш.', '`lots` са на grain по обособена позиция — не ги брой едно към едно срещу `contracts`.', diff --git a/apps/web/app/routes/contract.tsx b/apps/web/app/routes/contract.tsx index 8eb1175b..e3347461 100644 --- a/apps/web/app/routes/contract.tsx +++ b/apps/web/app/routes/contract.tsx @@ -222,7 +222,13 @@ export default function Contract({ loaderData }: Route.ComponentProps) {
Текуща стойност
{v.currentEur != null ? money(v.currentEur) : '—'} - {v.suspect &&
{UNVERIFIED_VALUE_LABEL}
} + {v.currentValueDoubled ? ( +
+ стойността изглежда двойно отчетена и не се показва +
+ ) : ( + v.suspect &&
{UNVERIFIED_VALUE_LABEL}
+ )} {v.deltaPct != null && (
{signedPct(v.deltaPct)} спрямо сключване
)} @@ -230,8 +236,10 @@ export default function Contract({ loaderData }: Route.ComponentProps) {
{v.suspect && (

- Показана е публикуваната стойност от източника, без СИГМА да я коригира. Виж{' '} - методология. + {v.currentValueDoubled + ? 'Текущата стойност изглежда двойно отчетена в източника и затова не се показва. ' + : 'Показана е публикуваната стойност от източника, без СИГМА да я коригира. '} + Виж методология.

)} {c.frameworkAwards != null && ( @@ -267,11 +275,32 @@ export default function Contract({ loaderData }: Route.ComponentProps) { {c.amendments.map((a, i) => ( {a.date ? longDate(a.date) : '—'} + {/* #305 residual: an uncorrectable double-count — the source's value_after is the + untrusted doubled figure, so show „—" and mark the row rather than a number we + can't stand behind. A `restated` row is the opposite: СИГМА corrected the doubled + total from the основание text, so we show the corrected number and flag that we + rewrote it. */} - {a.valueAfterEur != null ? moneyBare(a.valueAfterEur) : '—'} + {a.suspect ? ( + <> + — непотвърден тотал + + ) : a.valueAfterEur != null ? ( + <> + {moneyBare(a.valueAfterEur)} + {a.restated && ( + <> + {' '} + коригиран тотал + + )} + + ) : ( + '—' + )} - {a.deltaEur != null ? signedMoney(a.deltaEur) : '—'} + {!a.suspect && a.deltaEur != null ? signedMoney(a.deltaEur) : '—'} diff --git a/docs/README.md b/docs/README.md index 408599db..01cda08a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -11,6 +11,7 @@ - [`etl-architecture.md`](etl-architecture.md) — целевата ETL архитектура (RFC): предложение за състоянието и реда на изпълнение. - [`v1-implementation-plan.md`](v1-implementation-plan.md) — precompute слоят и пагинацията (защо rollup-и и keyset вместо per-request GROUP BY / OFFSET). - [`implementation-plans/286-ocds-amendment-unp.md`](implementation-plans/286-ocds-amendment-unp.md) — защо OCDS анексите не се свързват с договор (OCID вместо УНП) и планът за поправка през bridge-а `tender.id → УНП` + prefer-EOP dedup (#286). +- [`implementation-plans/305-amendment-value-double-count.md`](implementation-plans/305-amendment-value-double-count.md) — защо стойността на анекс се удвоява (ЦАИС ЕОП слага новия **тотал** в полето за промяна) и планът за откриване/поправка: `annex_total_suspect` флаг + текстова хеуристика за възстановяване на истинския тотал (#305). - [`integrity-gate.md`](integrity-gate.md) — reconciliation gate-ът: hard asserts върху тоталите при import/CI. - [`anomaly-report.md`](anomaly-report.md) — cross-row аномалии при опресняване: какво `value_flag` не хваща на ниво отделен договор. - [`deploy.md`](deploy.md) — деплой към Cloudflare: двата Worker-а (`sigma`, `sigma-etl`) и споделеният D1 per environment. diff --git a/docs/implementation-plans/305-amendment-value-double-count.md b/docs/implementation-plans/305-amendment-value-double-count.md new file mode 100644 index 00000000..7c67c492 --- /dev/null +++ b/docs/implementation-plans/305-amendment-value-double-count.md @@ -0,0 +1,119 @@ +# Implementation Plan: #305 — Amendment value double-count (a new *total* is added to the old value) + +## Executive Summary + +| Field | Value | +|---|---| +| Ticket | [midt-bg/sigma#305](https://github.com/midt-bg/sigma/issues/305) — labels `data-quality`, `etl`, `priority: high` | +| Problem | When an EOP annex announces a new **total** contract value, ЦАИС ЕОП puts that total in the *change* field, so `currentContractValue = lastContractValue + newTotal`. Sigma stores `value_after` verbatim, so the served value is **doubled** (the old value is counted twice). | +| Root cause | **Source data defect, faithfully stored.** `base.ts:313-315` maps `value_before ← lastContractValue`, `value_after ← currentContractValue`, `value_delta ← contractValueDifference` with **no arithmetic**. The bug is that, for a subset of annexes, `contractValueDifference` (→ `value_delta`) holds the **new total**, not the increment — and the feed's `currentContractValue` is `before + that total`. Verified: `value_after = value_before + value_delta` on **100%** of delta-carrying rows on `sigma-dev`. | +| Why existing flags miss it | A double-count is only ~2× signed value. `value_flag = 'annex_suspect'` needs ≥5× aggregate (+ a ≥10× per-step); #299 the same; the estimate-based flags need ≥10×/≥200×. 2× < 5× ⇒ classed **`ok`** ⇒ enters the `amount_eur` canonical value base and inflates every rollup. (`normalize-raw.sql:882-948`.) | +| Scale (real corpus) | On the issue's 2020→2026 local rebuild: 7,335 price-raising annexes, **686 at ≥100% growth**, ~666 unflagged, **€475.4M**. Independently reproduced on `sigma-dev`: **686 at ≥100% growth**, ~526 unflagged. The named records (145652, 189325, 84818) confirm. | +| Complexity | Medium. Detection is the hard part (a source-text heuristic with false-positive risk); the plumbing (flag + exclude, then optional correct) mirrors the existing `annex_suspect` machinery. | +| Risk | Medium. A naive correction that trusts a text heuristic can mis-restate genuine >100% increases; a flag-only tier is safe and ships first. The signature has a **blind spot** (a new-total *lower* than the old value hides below +100%), so the fix must not be sold as complete. | +| Status | **Draft — investigated with real DB calls + code trace.** | + +> **Two independently-true facts frame the fix.** (1) The source is internally *consistent* — `value_after = value_before + value_delta` always — so the correction can be expressed purely as *"when `value_delta` is a **total**, the true `value_after` is `value_delta`, not `value_before + value_delta`."* (2) The defect is `value_flag = 'ok'`, so it is inside the aggregated value base; the minimal safe fix is to move it *out* of that base (a new verdict), exactly as #299 did for its case. + +--- + +## 1. Problem, verified on real data + +### 1a. The mechanism (code, on `main`) +- `packages/ingest/src/base.ts:313-315` — EOP annexes map three source keys straight to columns, no math: + - `value_before ← lastContractValue` + - `value_after ← currentContractValue` + - `value_delta ← contractValueDifference` (the only signed field) +- `scripts/derive-amendments.sql:147-155` (mirrored in `scripts/normalize-raw.sql:752-759`): `contracts.current_value` = the latest amendment's non-null `value_after`. So the doubled `value_after` becomes the served contract value. +- `scripts/promote-amendments.sql:42-44`: `value_before/after/delta` copied verbatim into served `amendments`. + +**Conclusion:** the doubled figure is not computed by Sigma; it arrives in `currentContractValue`. For the affected annexes the authority entered the **new total** into `contractValueDifference`, and the feed's `currentContractValue = lastContractValue + newTotal`. Sigma stores it faithfully. + +### 1b. The named records (queried on `sigma-dev`, read-only) + +| contract | source | value_before | value_after | value_delta | doubled? | +|---|---|---|---|---|---| +| 145652 (УНП 00010-2023-0006) | `eop:annexes:2024-06-21` | 442,000 | **981,240** | 539,240 | yes — `value_delta` (539,240) is the announced new total; true `value_after` = 539,240 | +| 189325 (УНП 00210-2024-0024) | `eop:annexes:2025-10-07` | 77,000,000 | **154,000,000** | 77,000,000 | yes — exact 2× (currency-change annex) | +| 84818 (УНП 00080-2023-0001) | `eop:annexes:2026-07-17` (EUR) | 76,769,540.87 | **153,539,081.74** | 76,769,540.87 | yes — exact 2× | + +Caveats found in verification: 84818 has **6** amendment rows (only the 2026-07-17 EUR annex is the doubled one — earlier BGN annexes are consistent); the issue treated it as one. Absolute counts differ from the issue because `sigma-dev` (31,543 amendments) is a superset of the issue's local rebuild (26,921). + +### 1c. The math signature and its blind spot +- `value_delta = value_after − value_before` holds on **100%** of delta-carrying rows (`sigma-dev`), i.e. the source is self-consistent. The defect is semantic: `value_delta` is sometimes a *total*, not an *increment*. +- A double-count where `newTotal ≥ before` produces growth **≥ 100%** (686 rows). The exactly-+100% subset (`value_after = 2×before`) is the "same total re-stated" / currency-change case. +- **Blind spot:** if `newTotal < before`, the double-count yields growth **< 100%** and hides among clean rows. The +100% line is a *safety threshold*, not a proof of cleanliness — the fix must say so. + +--- + +## 2. Why the existing flags don't catch it + +`scripts/normalize-raw.sql:882-948` (mirrored in `refresh-slice.sql`), evaluated top-down against `eff_eur = EUR(COALESCE(current_value, signing_value))`: +- `value_suspect`: `eff_eur > 2e9`, or `> 200 × proc_est_eur`, or the стотинки band — estimate-relative, ignores a 2× overrun. +- `annex_suspect` (`:937-945`, the #299/#248 rule): `current_value/signing_value ≥ 100`, **or** (`≥ 5` **and** a per-step `value_after ≥ 10 × value_before`). A double-count is ~2× signed ⇒ below 5×. +- `review`: `eff_eur ≥ 10 × proc_est_eur`. +- else `ok`. + +A ~2× inflation clears none of these gates → `ok` → `amount_eur` takes `COALESCE(current_value, signing_value)` (`normalize-raw.sql:818-830`), so the doubled value is summed everywhere. Pinned by `packages/db/src/value-flag-annex-step-sql.test.ts` (the 5× floor is the smallest firing case; 4.9× stays `ok`). + +**Downstream consumers currently inflated** (all via the shared `amount_eur` base — `precompute.sql:16-19`): contract-list totals/sort/buckets and CSV export (`queries/contracts.ts:318,61-62,432,475`); `company_totals.won_eur`, `authority_totals.spent_eur`, `home_totals.value_eur` (`precompute.sql:42-60`); the contract detail value strip and the **amendment timeline**, which reads `amendments.value_after` unrepaired (`queries/details.ts:451-460,670-684`). *(The `/anomalies` #239 and `/overruns` #171 signals named in the issue are not on `main` yet — they will inherit the fix once they land.)* + +--- + +## 3. Detection + +The correction hinges on classifying each price-raising annex as **increment** vs **total**. Layer the signals; never rely on free text alone. + +1. **Arithmetic gate (necessary, cheap, high-recall / low-precision):** `value_before > 0 AND value_after ≥ 2 × value_before` (equivalently `value_delta ≥ value_before`). A single annex whose *increment* is ≥ the entire prior value is implausible; a double-count always lands here. Catches the 686. Does **not** catch the sub-100% blind spot (out of scope for v1, documented). +2. **Text confirmation (raises precision):** the основание free-text carries a number equal to `value_delta` in a *total* context — keywords `обща|общата|крайна|краен|възлиза|става` near the figure (the issue found 355 such records). Available fields at the raw/derive stage: `raw_amendments.description` (`changeDescription`), `reason` (`changeReason`), `circumstances` (`changeReasonDescription`) — see §5 note. Parse Bulgarian number formats (`1 234 567,89` / `1234567.89`), compare to `value_delta` within a small relative tolerance. +3. **Currency-change tell (special-case, very high precision):** exactly-+100% rows whose text mentions `евро|валута|EUR|лева в евро` are currency re-denominations with the total doubled (e.g. 189325). Treat as confirmed total. + +Classification: +- **Confirmed total** = gate (1) AND (text (2) or (3)). → correct (Tier 2) and/or flag. +- **Suspected total** = gate (1) only (no text confirmation). → flag-only (Tier 1); do not silently rewrite the value. + +--- + +## 4. The fix + +Two tiers. Ship Tier 1 first (safe, immediate); Tier 2 is the higher-value correction and needs the text heuristic hardened by tests. + +### Tier 1 — Flag and exclude from aggregates (minimal, safe, ships first) +Mirror the `annex_suspect` machinery so the ~475M/€ inflation leaves every rollup immediately, without trusting any text parse. +- Add a new `value_flag` verdict, e.g. **`annex_total_suspect`**, in `scripts/normalize-raw.sql` (and the `refresh-slice.sql` mirror + its reconciliation re-flag), placed **before** the `ELSE 'ok'`: fires when the contract's current-value-driving annex satisfies the arithmetic gate (§3.1) and the source-consistency check (`value_after ≈ value_before + value_delta`). +- Route it through the existing suspect fallback: `amount_eur`/`trusted_native` fall back to `signing_value` (`normalize-raw.sql:818-830`) and `precompute.sql:36` NULLs `current_value_eur` — so these contracts drop out of totals/CSV/pages exactly like `annex_suspect`. +- Emit a diagnostic count (like #286's diagnostics) so the flagged volume is observable in the ETL log. +- **Per-amendment flag (the issue's specific gap):** the contract flag does not fix the timeline row. Add a per-amendment marker so `queries/details.ts` can render the row as "suspected re-stated total" and suppress its `+%`. Options: a `value_flag`/`total_restated` column on served `amendments` (schema migration + carry through `promote-amendments.sql`), or recompute the same predicate in the details query. Prefer the column (single source of truth, avoids duplicating the heuristic in TS). + +### Tier 2 — Correct the value (higher value, needs text confirmation) +For **confirmed totals** (§3), restate the amendment at the **raw/derive stage** (where `value_before`, `value_delta`, and all three text fields coexist — see §5): +- Because the source is self-consistent, the correction is simply **`value_after := value_delta`** (the announced new total) and **`value_delta := value_after_old − value_before`** *no* — restate as: `corrected_after = value_delta_source` (the total); `corrected_delta = corrected_after − value_before` (the true increase). Keep the raw source values immutable in `raw_amendments`; write corrected values on the way to served `amendments` (a `promote`/derive transform), plus a `total_restated = 1` marker. +- `contracts.current_value` then derives from the corrected `value_after`, so headline value, deltas, and the timeline are all right — not merely excluded. +- Keep Tier-1 flagging for the **suspected-but-unconfirmed** set (gate only, no text) so nothing inflates while remaining un-restated. + +### Fix location (decided) +Raw/derive stage, **not** ingest and **not** the served query layer: +- Ingest (`base.ts`) must stay a faithful mirror of the source (per `docs/etl.md` non-destructive-staging stance) — do not mutate `raw_amendments`. +- The correction/flag needs `value_before`, `value_delta`, and the основание text on one row, which is true in `raw_amendments` and consumed by `derive-amendments.sql` / `normalize-raw.sql` / `promote-amendments.sql`. Implement there; keep `derive-amendments.sql` and `refresh-slice.sql` in lockstep (a drift guard already exists for the #286 bridge block — extend the pattern). + +--- + +## 5. Schema / data note (blocking for Tier 2) +Only `description` survives to served `amendments`; `reason` and `circumstances` are dropped at `promote-amendments.sql:9-11` (served DDL `0000_init.sql:169-182`). The text heuristic therefore must run at the **raw/derive** stage where all three exist (`work-staging-schema.sql:178-180`). If any per-row flag or corrected value must be *visible* to the app, add the column(s) to the served `amendments` table (migration) and carry them through `promote-amendments.sql` + the `refresh-slice.sql` amendments promotion. + +--- + +## 6. Testing — proving the fix +1. **Unit — number/keyword parser** (`packages/ingest` or a new `packages/db` SQL-driven test): Bulgarian number formats; total-context keywords vs increment phrasing; the currency-change tell. Fixtures from the real examples (145652 "възлезе на 539 240.00 лв."; 189325 currency change; a genuine >100% *increment* that must NOT be corrected). +2. **End-to-end SQL** (extend `packages/db/src/refresh-slice.test.ts` / a new `amendments-total-suspect.test.ts`): run the real `derive-amendments.sql → normalize-raw.sql → promote-amendments.sql` and assert: (a) a confirmed-total annex is restated (`value_after = value_delta`, `current_value` correct, `total_restated = 1`); (b) a suspected-only annex is flagged `annex_total_suspect` and excluded from `amount_eur`; (c) a genuine >100% increment stays `ok` and untouched (guard against false positives); (d) the exactly-+100% currency case restates to no real growth. +3. **Flag-coverage regression**: assert the new verdict count on a seeded corpus, and that `value-flag-annex-step-sql.test.ts`'s existing cases are unchanged. +4. **Real-corpus before/after**: rebuild the 2020→2026 corpus (local work-DB, then optionally ship), and report flagged/corrected counts + the EUR removed from `company_totals`/`authority_totals`/`home_totals`. Target: the 686 (≥100%) restated or flagged, headline totals drop by the double-counted amount, zero genuine-increment regressions. + +--- + +## 7. Scope boundaries & risks +- **In scope:** EOP annexes (the sole driver — OCDS rows carry `value_after = NULL` and never drive `current_value`, per #286). +- **Blind spot (out of scope for v1, must be documented):** double-counts where the new total is *lower* than the old value (growth < 100%) — not detectable by the arithmetic gate. Text-only detection could reach them but at higher false-positive cost; defer. +- **False-positive risk (the main hazard):** a genuine annex that legitimately more-than-doubles a contract. Tier 1 only *flags* (reversible, excludes from aggregates); Tier 2 *rewrites* and must require text confirmation + be unit-tested against a real >100%-increment fixture. When uncertain, prefer flag over rewrite. +- **Adjacent / not this:** #299 (`c44a7ee`) and #248 handle the ≥10×/≥5× mis-keying case (kept); #245 (EUR double-conversion) and #304/#247 (стотинки) are separate. This defect is the sub-5× band those cannot reach. +- **Consumer follow-through:** once `/anomalies` (#239) and `/overruns` (#171) land, confirm they read a value base that already excludes/corrects these (they will, if they use `amount_eur` + `value_flag`). diff --git a/packages/api-contract/src/index.ts b/packages/api-contract/src/index.ts index d1d9f709..84009da8 100644 --- a/packages/api-contract/src/index.ts +++ b/packages/api-contract/src/index.ts @@ -252,6 +252,9 @@ export interface ContractValueTimeline { currentEur: number | null; deltaPct: number | null; // (current − signing) / signing, when both present suspect: boolean; // value_/annex_suspect/review → render with an unverified-value label + // annex_total_suspect → the current value is a KNOWN exact 2× double-count. currentEur is blanked (—) + // rather than shown as a labelled doubled figure — a known-wrong number is worse than an honest gap (#307). + currentValueDoubled: boolean; } export interface ContractLotRow { @@ -282,6 +285,8 @@ export interface AmendmentEntry { description: string | null; // recorded reason/notes, when the source carries them valueAfterEur: number | null; // the contract value after this annex deltaEur: number | null; // value_after − value_before + restated: boolean; // #305 Tier-2: value_after was text-corrected from a double-counted total + suspect: boolean; // #305 residual: an uncorrectable double-count — value_after/delta suppressed, row marked } export interface ContractDetail { diff --git a/packages/db/migrations/0000_init.sql b/packages/db/migrations/0000_init.sql index 90f98dac..57002fd3 100644 --- a/packages/db/migrations/0000_init.sql +++ b/packages/db/migrations/0000_init.sql @@ -125,13 +125,13 @@ CREATE TABLE contracts ( bids_received INTEGER, contract_kind TEXT, -- Доставки / Услуги / Строителство awarded_to_group INTEGER, -- this AWARD went to an обединение (per-contract, distinct from bidders.is_consortium) - value_flag TEXT NOT NULL DEFAULT 'ok', -- ok | review | value_low | value_suspect | annex_suspect (data-quality verdict; assigned in scripts/normalize-raw.sql) + value_flag TEXT NOT NULL DEFAULT 'ok', -- ok | review | value_low | value_suspect | annex_suspect | annex_total_suspect (data-quality verdict; assigned in scripts/normalize-raw.sql) date_flag TEXT NOT NULL DEFAULT 'ok', -- ok | signed_after_publication (non-destructive date-quality verdict) amount_eur REAL, -- canonical EUR, SAFE TO SUM; populated for all flags (value_suspect repaired to the procedure estimate); NULL only when no trustworthy EUR figure (FX-rateless foreign / value_suspect w/o estimate / no signing+current) fx_converted INTEGER NOT NULL DEFAULT 0, -- 1 = amount_eur came from a foreign-currency market rate fx_rate REAL, -- EUR per 1 unit of `currency` for foreign rows (amount × fx_rate = amount_eur) signing_value_eur REAL, -- signing_value in EUR (peg/fx); NULL for value_suspect — for the contract value timeline - current_value_eur REAL, -- current_value in EUR; NULL for value_suspect/annex_suspect (suspect annex suppressed) + current_value_eur REAL, -- current_value in EUR; NULL for value_suspect/annex_suspect/annex_total_suspect (suspect annex suppressed) lot_id TEXT, -- domain lot id ('lot:'||УНП||':'||raw) when the award is lot-scoped; soft-links lots(id) document_number TEXT, -- Номер на документ published_at TEXT, -- Публикуван на diff --git a/packages/db/migrations/0006_amendment_restated.sql b/packages/db/migrations/0006_amendment_restated.sql new file mode 100644 index 00000000..5f158067 --- /dev/null +++ b/packages/db/migrations/0006_amendment_restated.sql @@ -0,0 +1,17 @@ +-- #305 Tier-2 text-based value correction (packages/ingest/src/amendment-total.ts). Some ЦАИС ЕОП +-- annexes put the announced NEW TOTAL into the change field, doubling value_after. The основание text +-- resolves each: a restated total drives the corrected value_after (and current_value), a genuine +-- increment is confirmed correct. The served amendments row records the outcome so the UI can mark a +-- corrected row and the refresh-slice reconciliation can skip text-treated annexes when arithmetic-flagging. +-- +-- value_restated = 1 when value_after was rewritten to the text-confirmed true total, else 0. +-- value_treatment = the raw treatment label ('total_restated' / 'unchanged_restated' / +-- 'genuine_increment', NULL when the text carried no signal). Kept alongside +-- value_restated because the slice reconciliation re-classifies from the served +-- amendments and must skip confirmed-genuine increments (value_restated stays 0 there). +-- Additive columns. On the live stage DB (whose migration ledger is empty — base schema imported +-- out-of-band) these are applied by the column probe in .github/workflows/deploy.yml, which ALTERs only +-- when the column is missing; on a fresh ledger `d1 migrations apply` runs this file exactly once. SQLite +-- has no `ADD COLUMN IF NOT EXISTS`, so do not replay this file against a DB that already has the columns. +ALTER TABLE amendments ADD COLUMN value_restated INTEGER NOT NULL DEFAULT 0; +ALTER TABLE amendments ADD COLUMN value_treatment TEXT; diff --git a/packages/db/migrations/0007_amendment_value_suspect.sql b/packages/db/migrations/0007_amendment_value_suspect.sql new file mode 100644 index 00000000..faa1b195 --- /dev/null +++ b/packages/db/migrations/0007_amendment_value_suspect.sql @@ -0,0 +1,14 @@ +-- #305 residual: a contract flagged value_flag = 'annex_total_suspect' has its current_value excluded +-- from every aggregate, but the served amendments row still carried the DOUBLED value_after — so the +-- amendment timeline kept showing the untrusted figure. Only text-corrected rows (value_restated) had +-- their served value rewritten; the ~183 flag-only doubles (non-exact >2×, no основание signal) did not. +-- We do NOT know their true total, so we MARK the row and let the UI SUPPRESS the untrusted figure — +-- never invent a number. +-- +-- value_suspect = 1 when the served amendment is a suspected double-count NOT already text-treated +-- (value_treatment IS NULL), else 0. A restated/genuine row (value_treatment set) is +-- never also suspect. The UI blanks value_after/delta for a suspect row. +-- Additive column. Applied on the live stage DB by the column probe in .github/workflows/deploy.yml (ALTER +-- only when missing); on a fresh ledger `d1 migrations apply` runs it once. SQLite has no `ADD COLUMN IF +-- NOT EXISTS`, so do not replay this file against a DB that already has the column. +ALTER TABLE amendments ADD COLUMN value_suspect INTEGER NOT NULL DEFAULT 0; diff --git a/packages/db/src/amendments-ocds-link.test.ts b/packages/db/src/amendments-ocds-link.test.ts index af0eac30..73dd62a5 100644 --- a/packages/db/src/amendments-ocds-link.test.ts +++ b/packages/db/src/amendments-ocds-link.test.ts @@ -13,6 +13,9 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest'; const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); const initSchema = resolve(root, 'packages/db/migrations/0000_init.sql'); +// #305 Tier-2: promote-amendments.sql writes value_restated/value_treatment to served amendments. +const migration6 = resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'); +const migration7 = resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'); const workStagingSchema = resolve(root, 'scripts/work-staging-schema.sql'); const deriveAmendments = resolve(root, 'scripts/derive-amendments.sql'); const promoteAmendments = resolve(root, 'scripts/promote-amendments.sql'); @@ -72,6 +75,8 @@ beforeEach(() => { dir = mkdtempSync(resolve(tmpdir(), 'amendments-ocds-')); db = resolve(dir, 'work.sqlite'); readScript(db, initSchema); // served `amendments` + `contracts` + readScript(db, migration6); // #305 Tier-2 value_restated/value_treatment on served amendments + readScript(db, migration7); // #305 residual value_suspect on served amendments readScript(db, workStagingSchema); // raw_* staging // Two EOP procedures: T1 (contract 90029) already has an EOP annex; T2 (contract 55500) has NONE. diff --git a/packages/db/src/amendments-sql.test.ts b/packages/db/src/amendments-sql.test.ts index 2d161410..75ceea4b 100644 --- a/packages/db/src/amendments-sql.test.ts +++ b/packages/db/src/amendments-sql.test.ts @@ -16,6 +16,10 @@ import { AMENDMENTS_SQL } from './queries/details'; const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); const migration0 = resolve(root, 'packages/db/migrations/0000_init.sql'); +// #305 Tier-2: AMENDMENTS_SQL selects am.value_restated, added to served amendments by this migration. +const migration6 = resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'); +// #305 residual: AMENDMENTS_SQL also selects am.value_suspect, added by this migration. +const migration7 = resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'); function sqlite(dbPath: string, sql: string): string { return execFileSync('sqlite3', [dbPath], { input: sql, encoding: 'utf8' }); @@ -65,6 +69,8 @@ function withDb(fn: (dbPath: string) => T): T { const dbPath = resolve(dir, 'test.sqlite'); try { readScript(dbPath, migration0); + readScript(dbPath, migration6); + readScript(dbPath, migration7); return fn(dbPath); } finally { rmSync(dir, { recursive: true, force: true }); diff --git a/packages/db/src/amendments-total-restated.test.ts b/packages/db/src/amendments-total-restated.test.ts new file mode 100644 index 00000000..5909f710 --- /dev/null +++ b/packages/db/src/amendments-total-restated.test.ts @@ -0,0 +1,301 @@ +/// +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; + +// #305 Tier-2 text-based value correction. Some ЦАИС ЕОП annexes put the announced NEW TOTAL into the +// change field, so the feed's value_after is doubled. The основание-text heuristic (computed in TS +// ingest, packages/ingest/src/amendment-total.ts) classifies each annex and — because the correction is +// computed BEFORE the SQL runs — the raw row lands with value_treatment + value_after_restated already +// set. This suite simulates that ingest output and drives the REAL derive → normalize/refresh-slice → +// promote → precompute scripts in pipeline order on a real SQLite DB, asserting: +// (a) a total_restated annex drives current_value + served value_after with the corrected total and is +// NOT annex_total_suspect; +// (b) a genuine_increment annex is not flagged and keeps its (larger, correct) value_after; +// (c) an untreated doubled annex still gets the Tier-1 annex_total_suspect flag; +// (d) full-vs-slice parity for the restated contract. + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); +const schemaPath = resolve(root, 'packages/db/migrations/0000_init.sql'); +const migration1Path = resolve(root, 'packages/db/migrations/0001_flow_pairs_bidder_index.sql'); +const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); +const migration3Path = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); +const migration6Path = resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'); +const migration7Path = resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'); +const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); +const derivePath = resolve(root, 'scripts/derive-amendments.sql'); +const normalizePath = resolve(root, 'scripts/normalize-raw.sql'); +const promotePath = resolve(root, 'scripts/promote-amendments.sql'); +const precomputePath = resolve(root, 'scripts/precompute.sql'); +const refreshSlicePath = resolve(root, 'scripts/refresh-slice.sql'); + +const etlRuns = [ + ['normalize-raw', [derivePath, normalizePath, promotePath, precomputePath]], + ['refresh-slice', [derivePath, refreshSlicePath, precomputePath]], +] as const; + +function sqlite(dbPath: string, sql: string): string { + return execFileSync('sqlite3', [dbPath], { input: sql, encoding: 'utf8' }); +} + +function sqliteJson(dbPath: string, sql: string): T[] { + const out = execFileSync('sqlite3', ['-json', dbPath, sql], { encoding: 'utf8' }).trim(); + return out ? (JSON.parse(out) as T[]) : []; +} + +function readScript(dbPath: string, path: string): void { + execFileSync('sqlite3', ['-bail', dbPath], { + input: `PRAGMA foreign_keys=ON;\n.read ${path}\n`, + stdio: 'pipe', + }); +} + +function withEtlDb(label: string, run: (dbPath: string) => void): void { + const dir = mkdtempSync(resolve(tmpdir(), `sigma-totalrestated-${label}-`)); + const dbPath = resolve(dir, 'test.sqlite'); + try { + readScript(dbPath, schemaPath); + readScript(dbPath, migration1Path); + readScript(dbPath, migration2Path); + readScript(dbPath, migration3Path); + readScript(dbPath, migration6Path); + readScript(dbPath, migration7Path); + readScript(dbPath, stagingPath); + run(dbPath); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +const AUTH_EIK = '000695114'; +const BIDDER_EIK = '831646048'; + +interface AmendmentStep { + before: number | null; + after: number; + publishedAt: string; + treatment: string | null; + restatedAfter: number | null; +} + +interface Case { + unp: string; + signing: number; + steps: AmendmentStep[]; +} + +function seedContracts(dbPath: string, cases: Case[]): void { + const tenders = cases + .map( + (c) => + `('eop:tenders:${c.unp}', '2026-06-01T00:00:00Z', '${c.unp}', '${AUTH_EIK}', 'Тестов възложител', 'public', ${c.signing}, 'BGN')`, + ) + .join(',\n'); + const contracts = cases + .map( + (c) => + `('eop:contracts:${c.unp}', '2026-06-01T00:00:00Z', '${c.unp}', '${AUTH_EIK}', 'Тестов възложител', 'C-${c.unp}', '2026-06-01', ${c.signing}, 'BGN', '${BIDDER_EIK}', 'Тестов изпълнител')`, + ) + .join(',\n'); + sqlite( + dbPath, + `INSERT INTO raw_tenders + (source, fetched_at, unp, authority_eik, authority_name, authority_type, estimated_value, currency) + VALUES ${tenders}; + + INSERT INTO raw_contracts + (source, fetched_at, unp, authority_eik, authority_name, contract_number, + contract_date, signing_value, currency, contractor_eik, contractor_name) + VALUES ${contracts};`, + ); +} + +// Seed raw_amendments with value_treatment + value_after_restated already populated — simulating the TS +// ingest output (base.ts runs the amendment-total.ts heuristic before staging). +function seedAmendments(dbPath: string, cases: Case[]): void { + const rows = cases + .flatMap((c) => + c.steps.map( + (s, i) => + `('eop:annexes:${c.unp}', '2026-06-01T00:00:00Z', 'A-${c.unp}-${i + 1}', '${c.unp}', 'C-${c.unp}', '${s.publishedAt}', ${s.before ?? 'NULL'}, ${s.after}, 'BGN', ${s.treatment === null ? 'NULL' : `'${s.treatment}'`}, ${s.restatedAfter ?? 'NULL'})`, + ), + ) + .join(',\n'); + if (!rows) return; + sqlite( + dbPath, + `INSERT INTO raw_amendments + (source, fetched_at, document_number, unp, contract_number, published_at, + value_before, value_after, currency, value_treatment, value_after_restated) + VALUES ${rows};`, + ); +} + +interface ContractRow { + id: string; + value_flag: string; + current_value: number | null; +} + +const contractsByUnp = (dbPath: string): Map => { + const rows = sqliteJson( + dbPath, + `SELECT id, value_flag, current_value FROM contracts`, + ); + const out = new Map(); + for (const r of rows) out.set(r.id.split(':')[2]!, r); + return out; +}; + +interface AmendmentRow { + unp: string; + value_after: number | null; + value_delta: number | null; + value_restated: number | null; +} + +const amendmentsByUnp = (dbPath: string): Map => { + const rows = sqliteJson( + dbPath, + `SELECT unp, value_after, value_delta, value_restated FROM amendments`, + ); + const out = new Map(); + for (const r of rows) out.set(r.unp, r); + return out; +}; + +// (a) total_restated: doubled value_after (981240) but the основание text announced the true total +// (539240). Ingest set value_after_restated=539240, value_treatment='total_restated'. +const RESTATED: Case = { + unp: 'UNP-RESTATED', + signing: 442_000, + steps: [ + { + before: 442_000, + after: 981_240, + publishedAt: '2026-06-10', + treatment: 'total_restated', + restatedAfter: 539_240, + }, + ], +}; + +// (b) genuine_increment: value_after (60226.85) is a real ≥2× increase already applied; the text confirmed +// it, so ingest set value_treatment='genuine_increment' with restatedAfter NULL. Must NOT be flagged. +const GENUINE: Case = { + unp: 'UNP-GENUINE', + signing: 10_226.85, + steps: [ + { + before: 10_226.85, + after: 60_226.85, + publishedAt: '2026-06-10', + treatment: 'genuine_increment', + restatedAfter: null, + }, + ], +}; + +// (c) untreated doubled annex: no text signal, before ≈ signing, same currency → Tier-1 unchanged. +const UNTREATED: Case = { + unp: 'UNP-DOUBLE', + signing: 77_000_000, + steps: [ + { + before: 77_000_000, + after: 154_000_000, + publishedAt: '2026-06-10', + treatment: null, + restatedAfter: null, + }, + ], +}; + +describe('#305 Tier-2 text-based amendment value correction', () => { + for (const [label, scriptPaths] of etlRuns) { + it(`${label}: a total_restated annex drives the corrected total and is not flagged`, () => { + withEtlDb(label, (dbPath) => { + seedContracts(dbPath, [RESTATED]); + seedAmendments(dbPath, [RESTATED]); + for (const p of scriptPaths) readScript(dbPath, p); + + const contract = contractsByUnp(dbPath).get('UNP-RESTATED'); + expect(contract?.value_flag, 'restated annex is not arithmetic-flagged').toBe('ok'); + expect( + contract?.current_value, + 'current_value is the corrected total, NOT the doubled value', + ).toBe(539_240); + + const amendment = amendmentsByUnp(dbPath).get('UNP-RESTATED'); + expect(amendment?.value_after, 'served value_after is the corrected total').toBe(539_240); + expect(amendment?.value_delta, 'served delta is self-consistent (after − before)').toBe( + 539_240 - 442_000, + ); + expect(amendment?.value_restated, 'served row is marked restated').toBe(1); + }); + }); + + it(`${label}: a genuine_increment annex is not flagged and keeps its value_after`, () => { + withEtlDb(label, (dbPath) => { + seedContracts(dbPath, [GENUINE]); + seedAmendments(dbPath, [GENUINE]); + for (const p of scriptPaths) readScript(dbPath, p); + + const contract = contractsByUnp(dbPath).get('UNP-GENUINE'); + expect(contract?.value_flag, 'confirmed-genuine increment is not flagged').not.toBe( + 'annex_total_suspect', + ); + expect(contract?.current_value, 'current_value keeps the genuine increase').toBe(60_226.85); + + const amendment = amendmentsByUnp(dbPath).get('UNP-GENUINE'); + expect(amendment?.value_after, 'served value_after unchanged').toBe(60_226.85); + expect(amendment?.value_restated, 'genuine increment is not marked restated').toBe(0); + }); + }); + + it(`${label}: an untreated doubled annex still gets the Tier-1 annex_total_suspect flag`, () => { + withEtlDb(label, (dbPath) => { + seedContracts(dbPath, [UNTREATED]); + seedAmendments(dbPath, [UNTREATED]); + for (const p of scriptPaths) readScript(dbPath, p); + + const contract = contractsByUnp(dbPath).get('UNP-DOUBLE'); + expect(contract?.value_flag, 'untreated double is still flagged').toBe( + 'annex_total_suspect', + ); + + const amendment = amendmentsByUnp(dbPath).get('UNP-DOUBLE'); + expect(amendment?.value_restated, 'untreated double is not marked restated').toBe(0); + }); + }); + } + + it('full-vs-slice parity: the total_restated contract resolves identically on both paths', () => { + let full: ContractRow | undefined; + withEtlDb('parity-full', (dbPath) => { + seedContracts(dbPath, [RESTATED]); + seedAmendments(dbPath, [RESTATED]); + for (const p of [derivePath, normalizePath, promotePath, precomputePath]) + readScript(dbPath, p); + full = contractsByUnp(dbPath).get('UNP-RESTATED'); + }); + + let slice: ContractRow | undefined; + withEtlDb('parity-slice', (dbPath) => { + seedContracts(dbPath, [RESTATED]); + seedAmendments(dbPath, [RESTATED]); + for (const p of [derivePath, refreshSlicePath, precomputePath]) readScript(dbPath, p); + slice = contractsByUnp(dbPath).get('UNP-RESTATED'); + }); + + // Pin both paths to the concrete expected values — a cross-equality (full === slice) would also + // pass on dual-undefined, so assert against the literal on each path instead. + expect(full?.value_flag).toBe('ok'); + expect(slice?.value_flag).toBe('ok'); + expect(full?.current_value).toBe(539_240); + expect(slice?.current_value).toBe(539_240); + }); +}); diff --git a/packages/db/src/amendments-total-suspect.test.ts b/packages/db/src/amendments-total-suspect.test.ts new file mode 100644 index 00000000..2fa3c913 --- /dev/null +++ b/packages/db/src/amendments-total-suspect.test.ts @@ -0,0 +1,708 @@ +/// +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; + +// #305 single-annex value double-count: a driving annex whose value_after is >=2x its value_before is +// a data defect — ЗОП чл.116 caps a single amendment at +50%, so one step cannot legally more than +// double a contract. Such contracts get value_flag = 'annex_total_suspect' and fall back to +// signing_value, exactly like annex_suspect, so the doubled figure is excluded from every EUR +// aggregate. The ABS(value_after - current_value) tie binds the flag to the annex that DRIVES +// current_value: a doubled annex later superseded by a correct one is NOT flagged. +// +// The flag CASE and its value-fallback siblings live in copies across both derive paths (normalize-raw +// and refresh-slice) plus refresh-slice's reconciliation re-flag, so the rule is exercised through the +// REAL scripts in pipeline order on a real SQLite database. A copy left behind fails here. + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); +const schemaPath = resolve(root, 'packages/db/migrations/0000_init.sql'); +const migration1Path = resolve(root, 'packages/db/migrations/0001_flow_pairs_bidder_index.sql'); +const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); +const migration3Path = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); +// #305 Tier-2: served amendments gained value_restated/value_treatment (promote + refresh-slice write them). +const migration6Path = resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'); +// #305 residual: served amendments gained value_suspect (promote + refresh-slice write it). +const migration7Path = resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'); +const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); +const derivePath = resolve(root, 'scripts/derive-amendments.sql'); +const normalizePath = resolve(root, 'scripts/normalize-raw.sql'); +const promotePath = resolve(root, 'scripts/promote-amendments.sql'); +const precomputePath = resolve(root, 'scripts/precompute.sql'); +const refreshSlicePath = resolve(root, 'scripts/refresh-slice.sql'); + +// Real pipeline order (scripts/import.mjs): full derive runs derive-amendments → normalize-raw → +// promote-amendments → precompute; the slice derive runs derive-amendments → refresh-slice (which +// promotes the window's amendments itself) → precompute. precompute populates current_value_eur, which +// these assertions read. +const etlRuns = [ + ['normalize-raw', [derivePath, normalizePath, promotePath, precomputePath]], + ['refresh-slice', [derivePath, refreshSlicePath, precomputePath]], +] as const; + +function sqlite(dbPath: string, sql: string): string { + return execFileSync('sqlite3', [dbPath], { input: sql, encoding: 'utf8' }); +} + +function sqliteJson(dbPath: string, sql: string): T[] { + const out = execFileSync('sqlite3', ['-json', dbPath, sql], { encoding: 'utf8' }).trim(); + return out ? (JSON.parse(out) as T[]) : []; +} + +function readScript(dbPath: string, path: string): void { + execFileSync('sqlite3', ['-bail', dbPath], { + input: `PRAGMA foreign_keys=ON;\n.read ${path}\n`, + stdio: 'pipe', + }); +} + +function withEtlDb(label: string, run: (dbPath: string) => void): void { + const dir = mkdtempSync(resolve(tmpdir(), `sigma-totalsuspect-${label}-`)); + const dbPath = resolve(dir, 'test.sqlite'); + try { + readScript(dbPath, schemaPath); + readScript(dbPath, migration1Path); + readScript(dbPath, migration2Path); + readScript(dbPath, migration3Path); + readScript(dbPath, migration6Path); + readScript(dbPath, migration7Path); + readScript(dbPath, stagingPath); + run(dbPath); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +// Signing value in BGN; exactly 100_000 EUR at the fixed peg (÷1.95583) so the fallback reads directly. +const SIGNING_BGN = 195_583; +const AUTH_EIK = '000695114'; +const BIDDER_EIK = '831646048'; + +interface AmendmentStep { + before: number | null; + after: number; + publishedAt: string; +} + +interface Case { + unp: string; + signing: number; + steps: AmendmentStep[]; +} + +function seedContracts(dbPath: string, cases: Case[]): void { + const tenders = cases + .map( + (c) => + `('eop:tenders:${c.unp}', '2026-06-01T00:00:00Z', '${c.unp}', '${AUTH_EIK}', 'Тестов възложител', 'public', ${c.signing}, 'BGN')`, + ) + .join(',\n'); + const contracts = cases + .map( + (c) => + `('eop:contracts:${c.unp}', '2026-06-01T00:00:00Z', '${c.unp}', '${AUTH_EIK}', 'Тестов възложител', 'C-${c.unp}', '2026-06-01', ${c.signing}, 'BGN', '${BIDDER_EIK}', 'Тестов изпълнител')`, + ) + .join(',\n'); + sqlite( + dbPath, + `INSERT INTO raw_tenders + (source, fetched_at, unp, authority_eik, authority_name, authority_type, estimated_value, currency) + VALUES ${tenders}; + + INSERT INTO raw_contracts + (source, fetched_at, unp, authority_eik, authority_name, contract_number, + contract_date, signing_value, currency, contractor_eik, contractor_name) + VALUES ${contracts};`, + ); +} + +function seedAmendments(dbPath: string, cases: Case[]): void { + const rows = cases + .flatMap((c) => + c.steps.map( + (s, i) => + `('eop:annexes:${c.unp}', '2026-06-01T00:00:00Z', 'A-${c.unp}-${i + 1}', '${c.unp}', 'C-${c.unp}', '${s.publishedAt}', ${s.before ?? 'NULL'}, ${s.after}, 'BGN')`, + ), + ) + .join(',\n'); + if (!rows) return; + sqlite( + dbPath, + `INSERT INTO raw_amendments + (source, fetched_at, document_number, unp, contract_number, published_at, + value_before, value_after, currency) + VALUES ${rows};`, + ); +} + +interface Row { + id: string; + value_flag: string; + amount_eur: number | null; + current_value_eur: number | null; +} + +const rowsByUnp = (dbPath: string): Map => { + const rows = sqliteJson( + dbPath, + `SELECT id, value_flag, ROUND(amount_eur) AS amount_eur, + ROUND(current_value_eur) AS current_value_eur + FROM contracts`, + ); + const out = new Map(); + for (const r of rows) out.set(r.id.split(':')[2]!, r); + return out; +}; + +// #305 residual: seed raw_amendments including value_treatment + value_after_restated (the TS ingest +// output) so the served value_suspect / value_restated marks can be asserted end-to-end. +interface TreatedStep { + before: number | null; + after: number; + publishedAt: string; + treatment: string | null; + restatedAfter: number | null; +} +interface TreatedCase { + unp: string; + signing: number; + steps: TreatedStep[]; +} + +function seedTreatedContracts(dbPath: string, cases: TreatedCase[]): void { + seedContracts( + dbPath, + cases.map((c) => ({ unp: c.unp, signing: c.signing, steps: [] })), + ); +} + +function seedTreatedAmendments(dbPath: string, cases: TreatedCase[]): void { + const rows = cases + .flatMap((c) => + c.steps.map( + (s, i) => + `('eop:annexes:${c.unp}', '2026-06-01T00:00:00Z', 'A-${c.unp}-${i + 1}', '${c.unp}', 'C-${c.unp}', '${s.publishedAt}', ${s.before ?? 'NULL'}, ${s.after}, 'BGN', ${s.treatment === null ? 'NULL' : `'${s.treatment}'`}, ${s.restatedAfter ?? 'NULL'})`, + ), + ) + .join(',\n'); + if (!rows) return; + sqlite( + dbPath, + `INSERT INTO raw_amendments + (source, fetched_at, document_number, unp, contract_number, published_at, + value_before, value_after, currency, value_treatment, value_after_restated) + VALUES ${rows};`, + ); +} + +interface ServedAmendment { + unp: string; + value_after: number | null; + value_suspect: number | null; + value_restated: number | null; +} +const servedByUnp = (dbPath: string): Map => { + const rows = sqliteJson( + dbPath, + `SELECT unp, value_after, value_suspect, value_restated FROM amendments`, + ); + const out = new Map(); + for (const r of rows) out.set(r.unp, r); + return out; +}; + +// (a) flag-only double: no основание signal (value_treatment NULL), before ≈ signing, value_after = 3× +// before. We can't bridge the true total → mark value_suspect = 1, keep value_restated = 0, and the +// served value_after stays the untrusted figure (the UI blanks it; the number is never rewritten). +const FLAG_ONLY: TreatedCase = { + unp: 'UNP-FLAGONLY', + signing: 100, + steps: [ + { before: 100, after: 300, publishedAt: '2026-06-10', treatment: null, restatedAfter: null }, + ], +}; +// (b) total_restated: text-treated → value_restated = 1, value_suspect = 0. +const RESTATED: TreatedCase = { + unp: 'UNP-RESTATED', + signing: 442_000, + steps: [ + { + before: 442_000, + after: 981_240, + publishedAt: '2026-06-10', + treatment: 'total_restated', + restatedAfter: 539_240, + }, + ], +}; +// (c) genuine_increment: text-confirmed real increase → both marks 0. +const GENUINE: TreatedCase = { + unp: 'UNP-GENUINE', + signing: 10_226.85, + steps: [ + { + before: 10_226.85, + after: 60_226.85, + publishedAt: '2026-06-10', + treatment: 'genuine_increment', + restatedAfter: null, + }, + ], +}; + +describe('#305 residual: per-amendment value_suspect marker on the served row', () => { + for (const [label, scriptPaths] of etlRuns) { + it(`${label}: a flag-only double is marked value_suspect=1, value_restated=0, value_after untouched`, () => { + withEtlDb(label, (dbPath) => { + seedTreatedContracts(dbPath, [FLAG_ONLY, RESTATED, GENUINE]); + seedTreatedAmendments(dbPath, [FLAG_ONLY, RESTATED, GENUINE]); + for (const p of scriptPaths) readScript(dbPath, p); + + const served = servedByUnp(dbPath); + + const flagOnly = served.get('UNP-FLAGONLY'); + expect(flagOnly?.value_suspect, 'flag-only double is marked suspect').toBe(1); + expect(flagOnly?.value_restated, 'flag-only double is NOT restated').toBe(0); + expect(flagOnly?.value_after, 'served value_after is never rewritten').toBe(300); + + const restated = served.get('UNP-RESTATED'); + expect(restated?.value_restated, 'total_restated row is marked restated').toBe(1); + expect(restated?.value_suspect, 'a restated row is never also suspect').toBe(0); + + const genuine = served.get('UNP-GENUINE'); + expect(genuine?.value_suspect, 'genuine increment is not suspect').toBe(0); + expect(genuine?.value_restated, 'genuine increment is not restated').toBe(0); + }); + }); + } + + it('full-vs-slice parity: the flag-only double gets value_suspect=1 on both paths', () => { + let full: ServedAmendment | undefined; + withEtlDb('parity-full', (dbPath) => { + seedTreatedContracts(dbPath, [FLAG_ONLY]); + seedTreatedAmendments(dbPath, [FLAG_ONLY]); + for (const p of [derivePath, normalizePath, promotePath, precomputePath]) + readScript(dbPath, p); + full = servedByUnp(dbPath).get('UNP-FLAGONLY'); + }); + + let slice: ServedAmendment | undefined; + withEtlDb('parity-slice', (dbPath) => { + seedTreatedContracts(dbPath, [FLAG_ONLY]); + seedTreatedAmendments(dbPath, [FLAG_ONLY]); + for (const p of [derivePath, refreshSlicePath, precomputePath]) readScript(dbPath, p); + slice = servedByUnp(dbPath).get('UNP-FLAGONLY'); + }); + + // Pin both paths to the concrete expected value — a cross-equality (full === slice) would also pass + // on dual-undefined, so assert against the literal on each path instead. + expect(full?.value_suspect).toBe(1); + expect(slice?.value_suspect).toBe(1); + }); +}); + +describe('#305 annex_total_suspect single-annex value double-count', () => { + for (const [label, scriptPaths] of etlRuns) { + it(`${label}: flags a doubled driving annex and falls back to the signing value`, () => { + withEtlDb(label, (dbPath) => { + const cases: Case[] = [ + // (a) The bug: 77M signed, one annex reports value_after = 2x value_before (154M). ЗОП caps a + // single amendment at +50%, so this is a double-count defect: flag it and drop back to signing. + { + unp: 'UNP-DOUBLE', + signing: 77_000_000, + steps: [{ before: 77_000_000, after: 154_000_000, publishedAt: '2026-06-10' }], + }, + // (b) A genuine small increase (+30%): before 100 → after 130. Must stay 'ok' and keep its + // current value, guarding against false positives. + { + unp: 'UNP-SMALL', + signing: 100, + steps: [{ before: 100, after: 130, publishedAt: '2026-06-10' }], + }, + ]; + seedContracts(dbPath, cases); + seedAmendments(dbPath, cases); + for (const p of scriptPaths) readScript(dbPath, p); + + const rows = rowsByUnp(dbPath); + + // (a) flagged, value base falls back to signing (77M BGN ÷1.95583), NOT the doubled 154M. + const doubled = rows.get('UNP-DOUBLE'); + expect(doubled?.value_flag, 'doubled annex flagged').toBe('annex_total_suspect'); + const signingEur = Math.round(77_000_000 / 1.95583); + expect(doubled?.amount_eur, 'amount_eur falls back to signing').toBe(signingEur); + // Excluded from the current_value aggregate entirely. + expect(doubled?.current_value_eur, 'current_value_eur suppressed').toBeNull(); + + // (b) genuine +30% increase untouched: stays ok, value reflects current_value (130 BGN). + const small = rows.get('UNP-SMALL'); + expect(small?.value_flag, 'small increase stays ok').toBe('ok'); + expect(small?.amount_eur, 'small increase keeps current value').toBe( + Math.round(130 / 1.95583), + ); + expect(small?.current_value_eur).toBe(Math.round(130 / 1.95583)); + }); + }); + + it(`${label}: an old doubled annex superseded by a correct later annex is NOT flagged`, () => { + withEtlDb(label, (dbPath) => { + const cases: Case[] = [ + // (d) An early annex doubled (before 100 → after 200), but a LATER annex sets a correct, + // sub-2x current_value (before 200 → after 130). The doubled step no longer DRIVES + // current_value, so the ABS(...-current_value) tie must leave the contract 'ok'. + { + unp: 'UNP-SUPERSEDED', + signing: 100, + steps: [ + { before: 100, after: 200, publishedAt: '2026-06-10' }, + { before: 200, after: 130, publishedAt: '2026-06-20' }, + ], + }, + ]; + seedContracts(dbPath, cases); + seedAmendments(dbPath, cases); + for (const p of scriptPaths) readScript(dbPath, p); + + const row = rowsByUnp(dbPath).get('UNP-SUPERSEDED'); + expect(row?.value_flag, 'superseded double is not flagged').toBe('ok'); + // Served at the corrected current value (130 BGN), not the transient doubled 200. + expect(row?.amount_eur).toBe(Math.round(130 / 1.95583)); + expect(row?.current_value_eur).toBe(Math.round(130 / 1.95583)); + }); + }); + } + + it('full-vs-slice parity: the doubled contract gets the same value_flag on both paths', () => { + const cases: Case[] = [ + { + unp: 'UNP-DOUBLE', + signing: 77_000_000, + steps: [{ before: 77_000_000, after: 154_000_000, publishedAt: '2026-06-10' }], + }, + ]; + + let fullFlag: string | undefined; + withEtlDb('parity-full', (dbPath) => { + seedContracts(dbPath, cases); + seedAmendments(dbPath, cases); + for (const p of [derivePath, normalizePath, promotePath, precomputePath]) + readScript(dbPath, p); + fullFlag = rowsByUnp(dbPath).get('UNP-DOUBLE')?.value_flag; + }); + + let sliceFlag: string | undefined; + withEtlDb('parity-slice', (dbPath) => { + seedContracts(dbPath, cases); + seedAmendments(dbPath, cases); + for (const p of [derivePath, refreshSlicePath, precomputePath]) readScript(dbPath, p); + sliceFlag = rowsByUnp(dbPath).get('UNP-DOUBLE')?.value_flag; + }); + + expect(fullFlag).toBe('annex_total_suspect'); + expect(sliceFlag).toBe('annex_total_suspect'); + expect(fullFlag).toBe(sliceFlag); + }); +}); + +// #305 multi-annex residual: the doubled step is NOT always the first annex. When a later annex reports +// a new TOTAL added to an already-grown value, its value_before is the prior CUMULATIVE total (a preceding +// annex's value_after), not signing. The relaxed anchor flags these too; the ≥2× single-step gate (ЗОП +// чл.116) keeps slow legitimate climbs — whose later steps never reach 2× — untouched. +describe('#305 annex_total_suspect multi-annex value double-count', () => { + for (const [label, scriptPaths] of etlRuns) { + it(`${label}: flags a later-in-chain double whose value_before ties to a prior annex total, not signing`, () => { + withEtlDb(label, (dbPath) => { + const cases: Case[] = [ + // Chain: 1M signed → annex1 +40% (1.4M, legal, not a double) → annex2 doubles the 1.4M total to + // 2.8M. The driving annex's value_before (1.4M) equals the PRIOR annex's value_after, not signing + // (1M), so the old signing-only anchor missed it. Must now flag and fall back to signing. + { + unp: 'UNP-MULTI-DOUBLE', + signing: 1_000_000, + steps: [ + { before: 1_000_000, after: 1_400_000, publishedAt: '2026-06-10' }, + { before: 1_400_000, after: 2_800_000, publishedAt: '2026-06-20' }, + ], + }, + // Control: same shape but the later step is a legal +36% (1.4M → 1.9M), below 2×. The relaxed + // anchor matches value_before to the prior total, but the ≥2× gate must keep this 'ok'. + { + unp: 'UNP-MULTI-OK', + signing: 1_000_000, + steps: [ + { before: 1_000_000, after: 1_400_000, publishedAt: '2026-06-10' }, + { before: 1_400_000, after: 1_900_000, publishedAt: '2026-06-20' }, + ], + }, + ]; + seedContracts(dbPath, cases); + seedAmendments(dbPath, cases); + for (const p of scriptPaths) readScript(dbPath, p); + + const rows = rowsByUnp(dbPath); + + const multiDouble = rows.get('UNP-MULTI-DOUBLE'); + expect(multiDouble?.value_flag, 'later-in-chain double flagged').toBe( + 'annex_total_suspect', + ); + expect(multiDouble?.amount_eur, 'amount_eur falls back to signing').toBe( + Math.round(1_000_000 / 1.95583), + ); + expect(multiDouble?.current_value_eur, 'current_value_eur suppressed').toBeNull(); + + const multiOk = rows.get('UNP-MULTI-OK'); + expect(multiOk?.value_flag, 'legal <2x later step stays ok').toBe('ok'); + expect(multiOk?.amount_eur, 'ok row keeps current value').toBe( + Math.round(1_900_000 / 1.95583), + ); + expect(multiOk?.current_value_eur).toBe(Math.round(1_900_000 / 1.95583)); + }); + }); + } + + // The per-row value_suspect marker (served amendments) must also catch the later-in-chain double, so the + // UI blanks that specific annex row — not just the contract-level flag. + const MULTI_TREATED: TreatedCase = { + unp: 'UNP-MULTI-SUSPECT', + signing: 1_000_000, + steps: [ + { + before: 1_000_000, + after: 1_400_000, + publishedAt: '2026-06-10', + treatment: null, + restatedAfter: null, + }, + { + before: 1_400_000, + after: 2_800_000, + publishedAt: '2026-06-20', + treatment: null, + restatedAfter: null, + }, + ], + }; + + for (const [label, scriptPaths] of etlRuns) { + it(`${label}: marks value_suspect=1 on the later-in-chain doubled annex row, not the legal earlier one`, () => { + withEtlDb(label, (dbPath) => { + seedTreatedContracts(dbPath, [MULTI_TREATED]); + seedTreatedAmendments(dbPath, [MULTI_TREATED]); + for (const p of scriptPaths) readScript(dbPath, p); + + const served = sqliteJson<{ value_after: number; value_suspect: number }>( + dbPath, + `SELECT value_after, value_suspect FROM amendments + WHERE unp = 'UNP-MULTI-SUSPECT' ORDER BY value_after`, + ); + expect(served.length, 'both annex rows served').toBe(2); + // The legal +40% step (1.4M) is not suspect; the doubled step (2.8M) is. + const legal = served.find((r) => r.value_after === 1_400_000); + const doubled = served.find((r) => r.value_after === 2_800_000); + expect(legal?.value_suspect, 'legal earlier step not suspect').toBe(0); + expect(doubled?.value_suspect, 'later-in-chain double marked suspect').toBe(1); + }); + }); + } +}); + +// #305 NEW-HIGH-1 (multi-annex chain contamination): the double-count correction is per-row and does NOT +// propagate down a chain. A restated prior annex (doubled → corrected down) leaves a LATER annex still +// computed by the feed on the contaminated (raw, doubled) base. The later annex's own step ratio is +// legitimate (<2×) so the arithmetic gate misses it and the prior is text-treated (excluded) — yet +// current_value inherited the doubled total. The new branch flags it → signing fallback, on both paths. +describe('#305 NEW-HIGH-1 multi-annex chain contamination', () => { + // annex1 doubled 1M→2.4M, text-restated to 1.4M; annex2 is a real +15% the feed computed on the RAW 2.4M + // base (2.4M→2.76M). annex2's own ratio is 1.15× so the gate misses it; annex1 is treated so it is + // excluded. Without the fix, current_value serves the contaminated 2.76M. + const CONTAM: TreatedCase = { + unp: 'UNP-CHAIN-CONTAM', + signing: 1_000_000, + steps: [ + { + before: 1_000_000, + after: 2_400_000, + publishedAt: '2026-06-10', + treatment: 'total_restated', + restatedAfter: 1_400_000, + }, + { + before: 2_400_000, + after: 2_760_000, + publishedAt: '2026-06-20', + treatment: null, + restatedAfter: null, + }, + ], + }; + // Control: the same shape on an HONEST base — annex1 is a genuine +40% (no restatement), annex2 +15% on + // the clean 1.4M base. No treated prior, so the contamination branch must NOT fire; stays 'ok'. + const CLEAN: TreatedCase = { + unp: 'UNP-CHAIN-CLEAN', + signing: 1_000_000, + steps: [ + { + before: 1_000_000, + after: 1_400_000, + publishedAt: '2026-06-10', + treatment: null, + restatedAfter: null, + }, + { + before: 1_400_000, + after: 1_610_000, + publishedAt: '2026-06-20', + treatment: null, + restatedAfter: null, + }, + ], + }; + + for (const [label, scriptPaths] of etlRuns) { + it(`${label}: flags a later annex riding a restated prior's doubled base, and keeps a clean chain ok`, () => { + withEtlDb(label, (dbPath) => { + seedTreatedContracts(dbPath, [CONTAM, CLEAN]); + seedTreatedAmendments(dbPath, [CONTAM, CLEAN]); + for (const p of scriptPaths) readScript(dbPath, p); + + const rows = rowsByUnp(dbPath); + + const contam = rows.get('UNP-CHAIN-CONTAM'); + expect(contam?.value_flag, 'contaminated later annex flagged').toBe('annex_total_suspect'); + expect(contam?.amount_eur, 'falls back to signing, not the contaminated 2.76M').toBe( + Math.round(1_000_000 / 1.95583), + ); + expect(contam?.current_value_eur, 'contaminated current_value suppressed').toBeNull(); + + const clean = rows.get('UNP-CHAIN-CLEAN'); + expect(clean?.value_flag, 'honest two-step growth stays ok').toBe('ok'); + expect(clean?.current_value_eur).toBe(Math.round(1_610_000 / 1.95583)); + }); + }); + } + + it('full-vs-slice parity: the contaminated chain gets the same flag on both paths', () => { + let fullFlag: string | undefined; + withEtlDb('parity-full', (dbPath) => { + seedTreatedContracts(dbPath, [CONTAM]); + seedTreatedAmendments(dbPath, [CONTAM]); + for (const p of [derivePath, normalizePath, promotePath, precomputePath]) + readScript(dbPath, p); + fullFlag = rowsByUnp(dbPath).get('UNP-CHAIN-CONTAM')?.value_flag; + }); + + let sliceFlag: string | undefined; + withEtlDb('parity-slice', (dbPath) => { + seedTreatedContracts(dbPath, [CONTAM]); + seedTreatedAmendments(dbPath, [CONTAM]); + for (const p of [derivePath, refreshSlicePath, precomputePath]) readScript(dbPath, p); + sliceFlag = rowsByUnp(dbPath).get('UNP-CHAIN-CONTAM')?.value_flag; + }); + + expect(fullFlag).toBe('annex_total_suspect'); + expect(sliceFlag).toBe('annex_total_suspect'); + }); +}); + +// #305 84818-class: an EXACT single-step 2× (value_after ≈ 2× value_before) is the ЗОП чл.116 defect +// signature even when value_before anchors to NEITHER signing NOR a prior annex total (an orphan base) — +// as in real contract 84818, whose annex reports 76.77M → 153.54M on a base unrelated to the contract's +// signing. The gate flags it → signing fallback (EXCLUDE), without ever rewriting the value. +describe('#305 84818-class orphan exact-double', () => { + for (const [label, scriptPaths] of etlRuns) { + it(`${label}: flags an exact 2× on an orphan base, but leaves a non-exact orphan jump ok`, () => { + withEtlDb(label, (dbPath) => { + const cases: Case[] = [ + // Orphan exact 2×: value_before 90 000 ties neither signing (195 583) nor any prior annex, and + // value_after is exactly 2×. Flag → signing fallback (100 000 EUR), never the doubled 180 000. + { + unp: 'UNP-ORPHAN-EXACT', + signing: 195_583, + steps: [{ before: 90_000, after: 180_000, publishedAt: '2026-06-10' }], + }, + // Control: an orphan jump that is NOT an exact 2× (1.5×) is ambiguous — with no anchor and no + // exact-double signature it must stay ok (the relaxed rule is scoped to EXACT 2× only). + { + unp: 'UNP-ORPHAN-SMALL', + signing: 195_583, + steps: [{ before: 90_000, after: 135_000, publishedAt: '2026-06-10' }], + }, + ]; + seedContracts(dbPath, cases); + seedAmendments(dbPath, cases); + for (const p of scriptPaths) readScript(dbPath, p); + + const rows = rowsByUnp(dbPath); + + const orphan = rows.get('UNP-ORPHAN-EXACT'); + expect(orphan?.value_flag, 'orphan exact 2× flagged').toBe('annex_total_suspect'); + expect(orphan?.amount_eur, 'falls back to signing, not the doubled 180k').toBe( + Math.round(195_583 / 1.95583), + ); + expect(orphan?.current_value_eur, 'doubled current suppressed').toBeNull(); + + const small = rows.get('UNP-ORPHAN-SMALL'); + expect(small?.value_flag, 'non-exact orphan jump stays ok').toBe('ok'); + expect(small?.current_value_eur).toBe(Math.round(135_000 / 1.95583)); + }); + }); + } +}); + +// #305 NEW-HIGH-2 (reconciliation parity): the slice reconciliation reads the CUMULATIVE served +// `amendments`, whose value_after is RESTATED, while the full path anchors on RAW values. A restated prior +// annex used to flip the anchor's "prev not itself a double" test (restated 1.4M < 2×1.2M passes; the raw +// 2.4M would fail), flagging on the slice but not on the full rebuild. The `prev.value_restated = 0` guard +// restores parity: both paths reach the same verdict for a restated-prior + doubled-later chain. +describe('#305 NEW-HIGH-2 slice reconciliation parity', () => { + // annex1 doubled 1.2M→2.4M, restated to 1.4M; annex2 grows the RESTATED 1.4M to 2.9M (a ≥2× step, but + // deliberately NOT an exact 2× so the 84818-class rule doesn't fire and mask the guard under test). On the + // full (raw) path annex2's value_before (1.4M) anchors to neither signing (1M) nor the raw prior total + // (2.4M) → 'ok'. Pre-guard the slice reconciliation matched the restated 1.4M and flagged — the flip. + // Post-guard (prev.value_restated = 0): 'ok', matching the full rebuild. + const FLIP: TreatedCase = { + unp: 'UNP-RECON-FLIP', + signing: 1_000_000, + steps: [ + { + before: 1_200_000, + after: 2_400_000, + publishedAt: '2026-06-10', + treatment: 'total_restated', + restatedAfter: 1_400_000, + }, + { + before: 1_400_000, + after: 2_900_000, + publishedAt: '2026-06-20', + treatment: null, + restatedAfter: null, + }, + ], + }; + + it('full and slice agree on a restated-prior + doubled-later chain (no flag flip)', () => { + let fullFlag: string | undefined; + withEtlDb('parity-full', (dbPath) => { + seedTreatedContracts(dbPath, [FLIP]); + seedTreatedAmendments(dbPath, [FLIP]); + for (const p of [derivePath, normalizePath, promotePath, precomputePath]) + readScript(dbPath, p); + fullFlag = rowsByUnp(dbPath).get('UNP-RECON-FLIP')?.value_flag; + }); + + let sliceFlag: string | undefined; + withEtlDb('parity-slice', (dbPath) => { + seedTreatedContracts(dbPath, [FLIP]); + seedTreatedAmendments(dbPath, [FLIP]); + for (const p of [derivePath, refreshSlicePath, precomputePath]) readScript(dbPath, p); + sliceFlag = rowsByUnp(dbPath).get('UNP-RECON-FLIP')?.value_flag; + }); + + // The canonical full-rebuild verdict, matched by the slice (pre-guard the slice flipped to suspect). + expect(sliceFlag).toBe(fullFlag); + }); +}); diff --git a/packages/db/src/contractor-identity-sql.test.ts b/packages/db/src/contractor-identity-sql.test.ts index efd769fe..e47c775a 100644 --- a/packages/db/src/contractor-identity-sql.test.ts +++ b/packages/db/src/contractor-identity-sql.test.ts @@ -16,6 +16,16 @@ const migration3 = readFileSync( resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'), 'utf8', ); +// #305 Tier-2: served amendments gained value_restated/value_treatment (promote + refresh-slice write them). +const migration6 = readFileSync( + resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'), + 'utf8', +); +// #305 residual: served amendments gained value_suspect (promote + refresh-slice write it). +const migration7 = readFileSync( + resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'), + 'utf8', +); const staging = readFileSync(resolve(root, 'scripts/work-staging-schema.sql'), 'utf8'); const normalize = readFileSync(resolve(root, 'scripts/normalize-raw.sql'), 'utf8'); const precompute = readFileSync(resolve(root, 'scripts/precompute.sql'), 'utf8'); @@ -67,6 +77,8 @@ function build(path: 'normalize' | 'refresh'): DatabaseSync { db.exec(schema); db.exec(migration2); db.exec(migration3); + db.exec(migration6); + db.exec(migration7); db.exec(staging); db.exec(seed); if (path === 'normalize') { diff --git a/packages/db/src/etl-entity-canonicalization-sql.test.ts b/packages/db/src/etl-entity-canonicalization-sql.test.ts index df67b27e..7a682ffa 100644 --- a/packages/db/src/etl-entity-canonicalization-sql.test.ts +++ b/packages/db/src/etl-entity-canonicalization-sql.test.ts @@ -11,6 +11,9 @@ const schemaPath = resolve(root, 'packages/db/migrations/0000_init.sql'); const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); // refresh-slice.sql's officials block reads interest_links (0003); build it so the script doesn't fail. const migration3Path = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); +// #305 Tier-2: served amendments gained value_restated/value_treatment (promote + refresh-slice write them). +const migration6Path = resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'); +const migration7Path = resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'); const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); const etlPaths = [ ['normalize-raw', resolve(root, 'scripts/normalize-raw.sql')], @@ -40,6 +43,8 @@ function withEtlDb(label: string, run: (dbPath: string) => void): void { readScript(dbPath, schemaPath); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration6Path); + readScript(dbPath, migration7Path); readScript(dbPath, stagingPath); run(dbPath); } finally { diff --git a/packages/db/src/queries/details.test.ts b/packages/db/src/queries/details.test.ts index 9371762f..014ffa8e 100644 --- a/packages/db/src/queries/details.test.ts +++ b/packages/db/src/queries/details.test.ts @@ -195,9 +195,35 @@ describe('getContract', () => { expect(detail?.value.suspect).toBe(true); expect(detail?.value.signingEur).toBe(256.49); expect(detail?.value.currentEur).toBe(flag === 'annex_suspect' ? 1025.96 : 256.49); + expect(detail?.value.currentValueDoubled).toBe(false); } }); + it('#307 blanks the current value for a KNOWN 2× double-count (annex_total_suspect)', async () => { + const detail = await getContract( + fakeDb( + { + ...baseContractRow, + signing_value: 256.49, + current_value: 512.98, // the doubled native figure — must NOT resurface + signing_value_eur: 256.49, + current_value_eur: null, // excluded from aggregates upstream + value_flag: 'annex_total_suspect', + }, + [], + ), + 'c:1', + ); + + expect(detail?.value.suspect).toBe(true); + expect(detail?.value.currentValueDoubled).toBe(true); + // Blanked (—), never the doubled 512.98 nor a fabricated fallback. + expect(detail?.value.currentEur).toBeNull(); + expect(detail?.value.deltaPct).toBeNull(); + // The trustworthy signing value is still shown. + expect(detail?.value.signingEur).toBe(256.49); + }); + // Exercises the real cohort path end-to-end (baseContractRow is clean-value, CPV '72', amount 5000). // Guards the argument order into contractCohort: swapping value_flag ↔ division would make it return // null and this would fail. @@ -351,6 +377,38 @@ describe('getContract', () => { }); }); + it('#305 residual: suppresses value_after and delta for a suspect (uncorrectable double-count) annex', async () => { + const detail = await getContract( + fakeDb( + { ...baseContractRow, contract_number: 'C-6' }, + [], + [ + { + value_before: 1000, + value_after: 3000, // the source's untrusted doubled/tripled total + value_delta: 2000, + currency: 'EUR', + published_at: '2024-03-01', + document_number: 'A1', + description: 'Изменение на стойността', + value_restated: 0, + value_suspect: 1, + fx_rate: null, + }, + ], + ), + 'c:1', + ); + + expect(detail?.amendments[0]).toMatchObject({ + valueAfterEur: null, // suppressed — we don't stand behind the doubled figure + deltaEur: null, + suspect: true, + restated: false, + description: 'Изменение на стойността', // description still shown + }); + }); + it('converts foreign-currency amendments to EUR via the annex fx rate', async () => { const detail = await getContract( fakeDb( diff --git a/packages/db/src/queries/details.ts b/packages/db/src/queries/details.ts index 7a211d8d..9786347e 100644 --- a/packages/db/src/queries/details.ts +++ b/packages/db/src/queries/details.ts @@ -431,6 +431,8 @@ interface AmendmentRow { published_at: string | null; document_number: string | null; description: string | null; + value_restated: number | null; + value_suspect: number | null; fx_rate: number | null; } @@ -454,7 +456,8 @@ export const AMENDMENTS_SQL = `SELECT am.value_before, am.value_after, am.value_ WHERE f.base_currency = am.currency AND f.rate_date <= am.published_at AND f.rate_date >= date(am.published_at, '-10 days') - ORDER BY f.rate_date DESC LIMIT 1) AS fx_rate + ORDER BY f.rate_date DESC LIMIT 1) AS fx_rate, + am.value_restated, am.value_suspect FROM amendments am WHERE am.unp = ? AND am.contract_number = ? ORDER BY am.published_at, am.id`; @@ -542,14 +545,20 @@ export async function getContract( const suspect = r.value_flag === 'value_suspect' || r.value_flag === 'annex_suspect' || + r.value_flag === 'annex_total_suspect' || r.value_flag === 'review' || r.value_flag === 'value_low'; const dateSuspect = r.date_flag === 'signed_after_publication'; + // #307 — annex_total_suspect is a KNOWN exact 2× double-count in current_value. Its current_value_eur is + // already NULL (excluded from aggregates), so the native fallback below would resurface the doubled figure + // under an "unverified" label. Blank it instead: a known-wrong number is worse than an honest gap. + const currentValueDoubled = r.value_flag === 'annex_total_suspect'; const signingEur = r.signing_value_eur ?? eurFromNative(r.signing_value, r.contract_currency, r.fx_rate); - const currentRaw = - r.current_value_eur ?? - eurFromNative(r.current_value, r.current_value_currency || r.contract_currency, r.fx_rate); + const currentRaw = currentValueDoubled + ? null + : (r.current_value_eur ?? + eurFromNative(r.current_value, r.current_value_currency || r.contract_currency, r.fx_rate)); const procedureEstimatedEur = eurFromNative( r.estimated_value, r.tender_currency, @@ -606,12 +615,13 @@ export async function getContract( estimatedEur: currentLotEstimatedEur ?? procedureEstimatedEur, procedureEstimatedEur, signingEur, - currentEur: currentRaw ?? signingEur, + currentEur: currentValueDoubled ? null : (currentRaw ?? signingEur), deltaPct: !suspect && currentRaw != null && signingEur != null && signingEur !== 0 ? (currentRaw - signingEur) / signingEur : null, suspect, + currentValueDoubled, }; const authority: ContractParty = { @@ -670,16 +680,28 @@ export async function getContract( const amendments: ContractDetail['amendments'] = amendmentRows.results.map((am) => { const beforeEur = eurFromNative(am.value_before, am.currency, am.fx_rate); const afterEur = eurFromNative(am.value_after, am.currency, am.fx_rate); + // #305 residual: a suspected double-count we could NOT correct from the основание text. Its + // value_after is the untrusted doubled figure, so suppress it (and the derived delta) rather than + // show a number we can't stand behind — the UI marks the row „непотвърден тотал". + const suspect = am.value_suspect === 1; return { date: am.published_at, documentNumber: am.document_number, description: am.description?.trim() || null, - valueAfterEur: afterEur, + valueAfterEur: suspect ? null : afterEur, // Compute delta from the SAME before/after we display, so the row is self-consistent (after − // before == delta) even when the source's recorded value_delta disagrees with them. When only // one of before/after is known, the recorded delta can't be reconciled against valueAfterEur — // show „—" rather than a figure that might not add up. - deltaEur: beforeEur != null && afterEur != null ? afterEur - beforeEur : null, + deltaEur: suspect + ? null + : beforeEur != null && afterEur != null + ? afterEur - beforeEur + : null, + // #305 Tier-2: the served value_after was rewritten from the основание text (a double-count total + // restated to the true value) — let the UI mark the corrected row. + restated: am.value_restated === 1, + suspect, }; }); diff --git a/packages/db/src/refresh-slice.test.ts b/packages/db/src/refresh-slice.test.ts index 2ab6831d..3685e1e4 100644 --- a/packages/db/src/refresh-slice.test.ts +++ b/packages/db/src/refresh-slice.test.ts @@ -13,6 +13,10 @@ const migration1Path = resolve(root, 'packages/db/migrations/0001_flow_pairs_bid const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); // refresh-slice.sql / precompute.sql officials block reads interest_links (0003) — build it in every chain. const migration3Path = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); +// #305 Tier-2: served amendments gained value_restated/value_treatment (refresh-slice promotes them). +const migration6Path = resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'); +// #305 residual: served amendments gained value_suspect (refresh-slice promotes it). +const migration7Path = resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'); const refreshSlicePath = resolve(root, 'scripts/refresh-slice.sql'); const normalizePath = resolve(root, 'scripts/normalize-raw.sql'); const deriveAmendmentsPath = resolve(root, 'scripts/derive-amendments.sql'); @@ -185,6 +189,8 @@ function initWorkDb(dbPath: string): void { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration6Path); + readScript(dbPath, migration7Path); readScript(dbPath, workStagingSchemaPath); } @@ -576,6 +582,8 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration6Path); + readScript(dbPath, migration7Path); readScript(dbPath, workStagingSchemaPath); seedEopBaseDay(dbPath); @@ -658,6 +666,8 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration6Path); + readScript(dbPath, migration7Path); readScript(dbPath, workStagingSchemaPath); // An EOP procedure (tender + base contract) with УНП UNP-SLICE / tender.id TENDER-SLICE. An @@ -841,6 +851,8 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration6Path); + readScript(dbPath, migration7Path); readScript(dbPath, workStagingSchemaPath); seedEopOnlySharedNumber(dbPath); readScript(dbPath, refreshSlicePath); @@ -892,6 +904,8 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration6Path); + readScript(dbPath, migration7Path); readScript(dbPath, workStagingSchemaPath); sqlite( dbPath, @@ -943,6 +957,8 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration6Path); + readScript(dbPath, migration7Path); readScript(dbPath, workStagingSchemaPath); sqlite( dbPath, @@ -1080,6 +1096,8 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration6Path); + readScript(dbPath, migration7Path); readScript(dbPath, workStagingSchemaPath); sqlite( dbPath, diff --git a/packages/db/src/value-flag-annex-step-sql.test.ts b/packages/db/src/value-flag-annex-step-sql.test.ts index ea9f91f5..bfa8af32 100644 --- a/packages/db/src/value-flag-annex-step-sql.test.ts +++ b/packages/db/src/value-flag-annex-step-sql.test.ts @@ -26,6 +26,9 @@ const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); const schemaPath = resolve(root, 'packages/db/migrations/0000_init.sql'); const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); const migration3Path = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); +// #305 Tier-2: served amendments gained value_restated/value_treatment (promote + refresh-slice write them). +const migration6Path = resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'); +const migration7Path = resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'); const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); const derivePath = resolve(root, 'scripts/derive-amendments.sql'); const promotePath = resolve(root, 'scripts/promote-amendments.sql'); @@ -60,6 +63,8 @@ function withEtlDb(label: string, run: (dbPath: string) => void): void { readScript(dbPath, schemaPath); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration6Path); + readScript(dbPath, migration7Path); readScript(dbPath, stagingPath); run(dbPath); } finally { diff --git a/packages/db/src/value-flag-stotinki-sql.test.ts b/packages/db/src/value-flag-stotinki-sql.test.ts index 2fc97624..d2fe07e5 100644 --- a/packages/db/src/value-flag-stotinki-sql.test.ts +++ b/packages/db/src/value-flag-stotinki-sql.test.ts @@ -20,6 +20,9 @@ const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); const schemaPath = resolve(root, 'packages/db/migrations/0000_init.sql'); const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); const migration3Path = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); +// #305 Tier-2: served amendments gained value_restated/value_treatment (promote + refresh-slice write them). +const migration6Path = resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'); +const migration7Path = resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'); const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); const etlPaths = [ ['normalize-raw', resolve(root, 'scripts/normalize-raw.sql')], @@ -49,6 +52,8 @@ function withEtlDb(label: string, run: (dbPath: string) => void): void { readScript(dbPath, schemaPath); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration6Path); + readScript(dbPath, migration7Path); readScript(dbPath, stagingPath); run(dbPath); } finally { diff --git a/packages/ingest/src/amendment-total.test.ts b/packages/ingest/src/amendment-total.test.ts new file mode 100644 index 00000000..ff1f9cef --- /dev/null +++ b/packages/ingest/src/amendment-total.test.ts @@ -0,0 +1,348 @@ +// #305 — the value-double-count text heuristic, tested against REAL основание text pulled from the live +// corpus (contracts 145652, 189325, 84818, 108677, 79382, 113291, 103903). The hazard is false positives, +// so the controls (genuine increment, uncorrectable, normal increase) matter as much as the hits. +import { describe, expect, it } from 'vitest'; +import { + classifyAmendmentValue, + restatedValueAfter, + isGenuineIncrement, + type AmendmentValueInput, +} from './amendment-total'; + +const mk = ( + valueBefore: number, + valueAfter: number, + valueDelta: number, + currency: string, + ...texts: string[] +): AmendmentValueInput => ({ valueBefore, valueAfter, valueDelta, currency, texts }); + +describe('#305 amendment value double-count heuristic', () => { + it('restates a total announced as "…на " (145652)', () => { + const t = classifyAmendmentValue( + mk( + 442000, + 981240, + 539240, + 'BGN', + 'относно актуализиране стойността на договора … общата стойност на договора ще възлезе на 539 240.00 лв. без ДДС', + ), + ); + expect(t).toEqual({ kind: 'total_restated', correctedAfter: 539240 }); + }); + + it('restates a total announced as "…от X на " (79382, 113291 family)', () => { + expect( + restatedValueAfter( + mk( + 197720, + 484414, + 286694, + 'BGN', + 'Общата стойност на договор № 447 се променя от 197 720.00 лева без ДДС на 286 694,00 (двеста осемдесет и шест хиляди) лева без ДДС', + ), + ), + ).toBe(286694); + expect( + restatedValueAfter( + mk( + 13662405.12, + 28356190.64, + 14693785.52, + 'BGN', + 'в чл. 7 (1) от Договора общата цена за изпълнение предмета на договора се променя от 13 662 405,12 лв. без ДДС на 14 693 785,52 лв. без ДДС', + ), + ), + ).toBe(14693785.52); + }); + + it('does NOT restate "в размер на / ресурс / increment" phrasings — they name the change, not the total', () => { + // Real corpus 271148→650754: "…максималния ресурс за изменението в размер на 379 606.50" names the + // INCREMENT; restating value_after to it would understate a genuine >100% increase. Must be `none`. + expect( + classifyAmendmentValue( + mk( + 271147.5, + 650754, + 379606.5, + 'BGN', + 'Срокът се удължава до изчерпване на максималния ресурс за изменението в размер на 379 606.50 лв. без ДДС', + ), + ).kind, + ).toBe('none'); + // "…или сума в размер на 86 363.08" — the added-work amount, not the new contract total. + expect( + restatedValueAfter( + mk( + 71969.23, + 151662.82, + 79693.59, + 'BGN', + 'Общата стойност на договорените СМР се променя от 71 969.23 без ДДС или сума в размер на 79 693.59 лв.', + ), + ), + ).toBeNull(); + }); + + it('restates a currency re-denomination that doubled an unchanged total (189325)', () => { + const t = classifyAmendmentValue( + mk( + 77000000, + 154000000, + 77000000, + 'BGN', + 'Считано от 01.10.2025 г., отпечатваната върху акцизните бандероли продажна цена се променя от лева в евро.', + ), + ); + expect(t).toEqual({ kind: 'unchanged_restated', correctedAfter: 77000000 }); + }); + + it('does NOT touch a genuine increment announced as "…с " (108677)', () => { + const input = mk( + 10226.85, + 60226.85, + 50000, + 'EUR', + 'Увеличава се финансовият ресурс на договор № АО – 05 – 168 с 50 000 /петдесет хиляди/ евро без ДДС.', + ); + expect(isGenuineIncrement(input)).toBe(true); + expect(restatedValueAfter(input)).toBeNull(); + }); + + it('does NOT restate an exact-2× when the text announces a DIFFERENT total ("до 18 900") — flag, not rewrite (103903)', () => { + // #307: exact 2× (delta 15 120 = value_before) BUT the text says the value rose "до 18 900" — it is NOT + // unchanged. Neither the doubled 30 240 nor the halved 15 120 is the true total, and 18 900 is not + // recoverable here, so return `none` and let the arithmetic annex_total_suspect flag exclude the row. + const t = classifyAmendmentValue( + mk( + 15120, + 30240, + 15120, + 'BGN', + 'Прогнозната стойност по договора се увеличава от 15 120 лв. без ДДС до 18 900 лв. без ДДС', + ), + ); + expect(t).toEqual({ kind: 'none' }); + }); + + it('does NOT text-freely restate an exact-2× when the основание carries no restatement signal (84818)', () => { + // #307: restructuring note with no value/unchanged signal. A text-free halving could erase a legitimate + // ЗОП чл.116 ал.1 т.1 in-scope +100% (pre-announced option clause), so it must fall to the arithmetic + // annex_total_suspect flag (exclude), not be rewritten to the before-value. + const t = classifyAmendmentValue( + mk( + 76769540.87, + 153539081.74, + 76769540.87, + 'EUR', + 'Следните курсове за 22 пилота се преструктурират и се изпълняват в рамките на гаранционния период', + ), + ); + expect(t).toEqual({ kind: 'none' }); + }); + + it('restates an exact-2× administrative annex (non-value change) to the before-value', () => { + const t = classifyAmendmentValue( + mk( + 2685, + 5370, + 2685, + 'BGN', + 'Променя се упълномощеното лице по договора. Несъществени промени.', + ), + ); + expect(t).toEqual({ kind: 'unchanged_restated', correctedAfter: 2685 }); + }); + + it('does NOT text-freely restate an exact-2× on an outside-ЗОП exception contract', () => { + // ЗОП чл.116's +50% cap does not bind exception contracts, so an exact +100% there can be a genuine + // increase — the text-free rule 3 must stand down and let the arithmetic flag exclude (not rewrite) it. + const base = mk( + 2685, + 5370, + 2685, + 'BGN', + 'Променя се упълномощеното лице по договора. Несъществени промени.', + ); + expect(classifyAmendmentValue({ ...base, outsideZop: true }).kind).toBe('none'); + // …but the same row in-scope of ЗОП is still restated (guard is scoped to rule 3 only). + expect(classifyAmendmentValue({ ...base, outsideZop: false })).toEqual({ + kind: 'unchanged_restated', + correctedAfter: 2685, + }); + }); + + it('still applies the text-confirmed rules on an outside-ЗОП contract', () => { + // The double-count is a feed defect independent of ЗОП scope, so a text-confirmed total is corrected + // even for an exception contract — only the text-free exact-2× fallback is gated by outsideZop. + const total = { + ...mk( + 442000, + 981240, + 539240, + 'BGN', + 'общата стойност на договора ще възлезе на 539 240.00 лв.', + ), + outsideZop: true, + }; + expect(classifyAmendmentValue(total)).toEqual({ + kind: 'total_restated', + correctedAfter: 539240, + }); + const incr = { + ...mk(10226.85, 60226.85, 50000, 'EUR', 'Увеличава се ресурсът с 50 000 евро без ДДС.'), + outsideZop: true, + }; + expect(isGenuineIncrement(incr)).toBe(true); + }); + + it('ignores normal increases (< 2×) and non-self-consistent rows', () => { + expect(classifyAmendmentValue(mk(100, 130, 30, 'BGN', 'обща стойност на 130 лв.')).kind).toBe( + 'none', + ); + // a ≠ b + d ⇒ the double-count model does not apply + expect(classifyAmendmentValue(mk(100, 250, 100, 'BGN', 'обща стойност на 100')).kind).toBe( + 'none', + ); + }); + + it('requires the text figure to actually equal the delta (no coincidental match)', () => { + // delta 500000 appears nowhere as a total; a different figure 12345 does — must not restate. + expect( + restatedValueAfter(mk(400000, 900000, 500000, 'BGN', 'обща стойност на 12 345 лв.')), + ).toBeNull(); + }); + + it('does NOT restate a bare "…на " over a NON-monetary number (#307 HIGH-1 — days / article nos.)', () => { + // "…удължава на 200 дни": 200 is a day count that coincidentally == value_delta. Without a currency + // anchor around the figure it must stay `none`, never overwrite the published 300 with 200. + expect( + classifyAmendmentValue(mk(100, 300, 200, 'BGN', 'Срокът на договора се удължава на 200 дни.')) + .kind, + ).toBe('none'); + // An article number after "на" — non-monetary, must not restate. + expect( + classifyAmendmentValue( + mk(100, 300, 200, 'BGN', 'Договорът се изменя на 200 съгласно чл. 116 на ЗОП.'), + ).kind, + ).toBe('none'); + }); + + it('does NOT anchor a day-count on a currency token elsewhere in the sentence (#307 MONEY_AFTER window)', () => { + // "…удължава на 200 дни, стойността остава 100 лв.": 200 is a DAY count; the "лв." belongs to a + // different figure downstream. A non-monetary unit right after 200 must veto it, not restate 300→200. + expect( + classifyAmendmentValue( + mk(100, 300, 200, 'BGN', 'Срокът се удължава на 200 дни, стойността остава 100 лв.'), + ).kind, + ).toBe('none'); + expect( + classifyAmendmentValue( + mk( + 100, + 300, + 200, + 'BGN', + 'Срокът за изпълнение на договора се удължава на 200 дни, без промяна в договорената сума в лв.', + ), + ).kind, + ).toBe('none'); + }); + + it('vetoes the QUALIFIED day/quantity unit, not just the bare word (#307 review — работни/календарни дни class)', () => { + // The unit almost never comes bare in real annexes ("работни дни", "календарни дни", "200 (двеста) + // дни", "кв.м"). Each of these is a duration/quantity that coincidentally == value_delta; none may + // overwrite the published 300 with 200. Tests the error CLASS, not one literal sentence. + const dayCounts = [ + 'Срокът се удължава на 200 работни дни, стойността остава 100 лв.', + 'Срокът се удължава на 200 календарни дни, стойността остава 100 лв.', + 'Срокът се удължава на 200 работни дни, без промяна в договорената сума в лв.', + 'Срокът се удължава на 200 к.д., стойността остава 100 лв.', + 'Срокът се удължава на 200 (двеста) дни, стойността остава 100 лв.', + 'Срокът се удължава на 200 р.д., стойността остава 100 лв.', + 'Площта се увеличава на 200 кв.м, стойността остава 100 лв.', + 'Обемът се увеличава на 200 куб.м, стойността остава 100 лв.', + ]; + for (const text of dayCounts) { + expect(classifyAmendmentValue(mk(100, 300, 200, 'BGN', text)).kind).toBe('none'); + } + }); + + it('the wider unit veto does NOT swallow a real monetary total (#307 review — reverse direction)', () => { + // A qualified/adjacent-word unit veto must not fire on genuine money phrasings: the figure still + // restates to the announced total. Guards against the veto over-reaching. + const realTotals = [ + 'Общата стойност на договора възлиза на 200 лв. без ДДС.', + 'Общата стойност на договора възлиза на 200 лева.', + 'Новата обща стойност възлиза на 200 лв. за срок от 12 месеца.', + 'Общата стойност се увеличава на 200 лева месечно.', + 'Общата стойност възлиза на 200 лв. и срокът се удължава с 30 работни дни.', + ]; + for (const text of realTotals) { + expect(restatedValueAfter(mk(100, 300, 200, 'BGN', text))).toBe(200); + } + }); + + it('does NOT restate an exact-2× on a bare payment-in-euro clause (#307 в-евро narrowing)', () => { + // "Плащанията…се извършват в евро…" is a payment-currency clause, NOT an unchanged-value signal — it + // must not halve a real +100%. Only "X в евро" re-denomination phrasing may restate (see 189325). + const t = classifyAmendmentValue( + mk( + 250000, + 500000, + 250000, + 'BGN', + 'Плащанията по договора се извършват в евро по сметка на изпълнителя.', + ), + ); + expect(t).toEqual({ kind: 'none' }); + }); + + it('restates a bare "…на " only WHEN a currency unit follows the figure (#307 HIGH-1 anchor)', () => { + // Same "…на " shape as the days case, but a currency unit anchors it as money ⇒ genuine total. + expect( + restatedValueAfter( + mk(100, 300, 200, 'BGN', 'Общата стойност на договора се променя на 200 лв. без ДДС.'), + ), + ).toBe(200); + }); + + it('does NOT rewrite an exact-2× with empty / whitespace-only texts (#307 HIGH-2 repro)', () => { + const t = classifyAmendmentValue({ + valueBefore: 539240, + valueAfter: 1078480, + valueDelta: 539240, + currency: 'BGN', + texts: [null, '', ' '], + outsideZop: null, + }); + expect(t).toEqual({ kind: 'none' }); + }); + + it('does NOT rewrite an exact-2× when the text is unrelated to value (#307 HIGH-2 repro)', () => { + const t = classifyAmendmentValue({ + valueBefore: 250000, + valueAfter: 500000, + valueDelta: 250000, + currency: 'BGN', + outsideZop: false, + texts: ['Смяна на адреса за кореспонденция на изпълнителя.'], + }); + expect(t).toEqual({ kind: 'none' }); + }); + + it('parses a dot-thousands + comma-decimal figure "1.234,56" (#305 number-format recall)', () => { + // Mixed-separator total announced as "…на 1.234,56 лв." — the resolver must read 1234.56, not 1.23. + expect( + restatedValueAfter( + mk(700, 1934.56, 1234.56, 'BGN', 'Общата стойност на договора се променя на 1.234,56 лв.'), + ), + ).toBe(1234.56); + // …and the US ordering "1,234.56" resolves to the same value. + expect( + restatedValueAfter( + mk(700, 1934.56, 1234.56, 'BGN', 'Общата стойност на договора се променя на 1,234.56 лв.'), + ), + ).toBe(1234.56); + }); +}); diff --git a/packages/ingest/src/amendment-total.ts b/packages/ingest/src/amendment-total.ts new file mode 100644 index 00000000..fb5e4495 --- /dev/null +++ b/packages/ingest/src/amendment-total.ts @@ -0,0 +1,227 @@ +// #305 — amendment value double-count. ЦАИС ЕОП sometimes puts the announced NEW TOTAL contract value +// into the "change" field (`contractValueDifference` → `value_delta`), so the feed's +// `currentContractValue` (→ `value_after`) = `lastContractValue` + newTotal — the value is doubled. The +// source is internally consistent (`value_after = value_before + value_delta` at ~100% of rows), so the +// signal is semantic, not arithmetic: is `value_delta` an increment or a total? +// +// This module answers that from the основание free text, which the raw feed carries in three fields +// (changeDescription/changeReason/changeReasonDescription). The discriminator is the Bulgarian preposition +// in front of the figure: "на " (to N) / "възлиза/става/обща стойност" ⇒ N is a TOTAL; "с " (by N) / +// "увеличава се … с" ⇒ N is an INCREMENT. See docs/implementation-plans/305-amendment-value-double-count.md. +// +// Conservative by design: it only classifies when the text unambiguously confirms; otherwise it returns +// `none` and leaves the row to the arithmetic `annex_total_suspect` flag (Tier 1). It NEVER rewrites a +// value it cannot corroborate from text. Note: JS `\b`/`\w` are ASCII-only, so all boundaries/letters use +// Unicode (`\p{L}`, explicit non-letter boundary) with the `u` flag. + +export type AmendmentValueTreatment = + // The delta is an announced total; the true value_after is value_delta (double-count corrected). + | { kind: 'total_restated'; correctedAfter: number } + // An exact 2× (value_delta ≈ value_before): the "difference" field echoed the OLD value, so the value + // is unchanged and value_after was doubled onto itself; the true value_after is value_before. Covers + // currency re-denominations and non-value administrative annexes alike. + | { kind: 'unchanged_restated'; correctedAfter: number } + // The delta is a genuine increment already correctly applied — value_after is right; do NOT flag it. + | { kind: 'genuine_increment' } + // No text signal — leave to the arithmetic flag. + | { kind: 'none' }; + +export interface AmendmentValueInput { + valueBefore: number | null; + valueAfter: number | null; + valueDelta: number | null; + currency: string | null; + texts: Array; + // #305 — the text-free exact-2× rule leans on ЗОП чл.116 (a single amendment caps at +50%, so +100% is a + // defect not a real increase). чл.116 does NOT bind contracts procured outside ЗОП (exception contracts), + // where a genuine +100% is legal — so for those, only the text-confirmed rules may restate. NULL/false = + // in-scope of ЗОП (the safe default: apply the rule). + outsideZop?: boolean | null; +} + +const REL_TOL = 0.005; // 0.5% — the text figure must be the SAME number as value_delta, allowing rounding. +// #305 — capture a full number token that may group thousands with space/nbsp/narrow-nbsp OR with '.'/',' +// (BG "1.234,56", US "1,234.56"); normalizeBgNumber disambiguates the decimal mark below. The token must +// END on a digit so a trailing sentence period ("…100. Нов срок") is not swallowed into the number. +const NUMBER_RE = /\d[\d\u0020\u00a0\u202f.,]*\d|\d/g; +const WS = /[\s  ]/g; + +// A left boundary: start-of-window or a non-letter, non-digit character (Unicode-aware — Cyrillic is a +// letter). Keywords that end the "before the figure" window signal how the figure should be read. +const B = '(?:^|[^\\p{L}\\d])'; +const TOTAL_CTX = new RegExp( + `${B}(?:възлиз\\p{L}*|възлез\\p{L}*|става|обща\\p{L}*\\s+(?:стойност|цена)|крайн\\p{L}*\\s+(?:стойност|цена)|нов\\p{L}*\\s+(?:обща\\s+)?(?:стойност|цена)|на)\\s*$`, + 'iu', +); +// The figure sits right after "от " (the OLD value) or "с/със " (an increment) — not a total. +const NOT_TOTAL_CTX = new RegExp(`${B}(?:от|с|със)\\s*$`, 'iu'); +// "…с " / "…със " — N is an increment already applied. +const INCREMENT_CTX = new RegExp(`${B}(?:с|със)\\s*$`, 'iu'); +// A wider veto on the "…на " total match: Bulgarian "в размер на " ("in the amount of N"), +// "ресурс … в размер на N", "допълнителни … на обща стойност N" name the CHANGE/added-work amount, not +// the new contract total — restating value_after := N there would be wrong (verified on the real corpus). +// Checked over a wider window than NOT_TOTAL_CTX because these markers sit a few words before the figure. +const TOTAL_VETO = /(?:в\s+размер|ресурс\p{L}*|допълнителн\p{L}*|увеличени\p{L}*|намалени\p{L}*)/iu; + +// #307 — a total restatement needs a MONETARY anchor bracketing the figure. Bare "на " is not a money +// signal ("на" also precedes days, article numbers, quantities), so "…удължава на 200 дни" would otherwise +// rewrite the value with a day count. Accept the figure only when a value keyword sits immediately before +// it (MONEY_BEFORE) OR a currency unit follows it (MONEY_AFTER). On the real corpus the value keyword is +// usually far from the figure ("…ще възлезе на 539 240.00 лв."), so the currency unit after the number is +// the load-bearing anchor. No ASCII \b (Cyrillic). +const MONEY_BEFORE = /(?:стойност|цена)\p{L}*\s*$/iu; +const MONEY_AFTER = /(?:^|[^\p{L}])(?:лв\.?|лева|лев|bgn|eur|евро|euro|€|usd|\$)(?![\p{L}])/iu; +// #307 — MONEY_AFTER scans the whole ~60-char window, so a sentence that names both a term and a value +// ("…удължава на 200 дни, стойността остава 100 лв.") lets a downstream currency token anchor a figure +// that is actually a day count. A non-monetary unit sitting IMMEDIATELY after the figure (days, months, +// years, count, percent) overrides any currency further along: the figure is a duration/quantity, never +// the contract value. Anchored at ^ against the post-figure slice so only the immediate suffix counts. +// Real BG annexes almost never write the unit bare — the term is qualified ("работни дни", "календарни +// дни") — so allow one optional adjective word (and an optional spelled-out number in brackets, "200 +// (двеста) дни") between the figure and the unit, and cover area/volume/weight units too. Errs safe: a +// false veto only downgrades a row to `none`, dropping it to the arithmetic annex_total_suspect flag +// rather than publishing a substituted value. +const NON_MONEY_UNIT_AFTER = + /^\s*(?:\([^)]*\)\s*)?(?:\p{L}+\s+)?(?:дни|дн\.|к\.\s?д\.|р\.\s?д\.|месец\p{L}*|години|год\.|броя|бр\.|кв\.?\s?м|куб\.?\s?м|тона|литра|%|процент\p{L}*)/iu; + +// #307 — the exact-2× "unchanged" restatement (rule 3) may only fire WITH a positive textual signal that +// the value did not really change: a currency re-denomination that mechanically doubled the figure, or an +// explicit "unchanged / non-material" phrasing. Absent any signal the row returns `none` and falls to the +// arithmetic annex_total_suspect flag (exclude), rather than silently halving a possibly-legitimate +// ЗОП чл.116 ал.1 т.1 in-scope +100% (a pre-announced option clause `outsideZop` cannot model). +// #307 — the anchor is the "X в евро" re-denomination phrasing (`лев… в евро`), NOT a bare "в евро": a +// payment-currency clause ("Плащанията…се извършват в евро…") says nothing about an unchanged total and +// would silently halve a real doubling. The bare form was also redundant — the 189325 fixture +// ("…се променя от лева в евро") is already caught by the `лев… в евро` alternative. +const RESTATE_UNCHANGED_CTX = + /(?:лев\p{L}*\s+в\s+евро|деноминаци\p{L}*|не\s*се\s+промен\p{L}*|остава\p{L}*\s+непромен\p{L}*|без\s+промяна|несъществен\p{L}*)/iu; + +function normalizeBgNumber(raw: string): number | null { + let t = raw.replace(WS, ''); + // #305 — when BOTH '.' and ',' appear the number uses one as a thousands separator and the other as the + // decimal mark (BG "1.234,56" or US "1,234.56"). The LAST-occurring separator is the decimal; strip the + // other (thousands) and normalise the decimal to '.'. Single-separator numbers keep the existing + // ≤2-fraction-digit convention (the space-thousands corpus: "539 240.00", "286 694,00"). + if (t.includes('.') && t.includes(',')) { + const decimalChar = t.lastIndexOf('.') > t.lastIndexOf(',') ? '.' : ','; + const thousandsChar = decimalChar === '.' ? ',' : '.'; + t = t.split(thousandsChar).join(''); + if (decimalChar === ',') t = t.replace(',', '.'); + } + const m = t.match(/^(\d+)(?:[.,](\d{1,2}))?$/); + if (!m) return null; + const value = Number(m[2] ? `${m[1]}.${m[2]}` : m[1]); + return Number.isFinite(value) && value > 0 ? value : null; +} + +function approxEq(a: number, b: number): boolean { + return Math.abs(a - b) <= REL_TOL * Math.max(Math.abs(a), Math.abs(b)); +} + +// Does a figure ≈ `target` occur in `text` with the ~40 preceding chars matching `contextRe` and (when +// given) NOT matching `excludeRe`? Returns true on the first qualifying occurrence. +function figureInContext( + text: string, + target: number, + contextRe: RegExp, + excludeRe: RegExp | null, + wideVetoRe: RegExp | null = null, + requireMoneyAnchor = false, +): boolean { + NUMBER_RE.lastIndex = 0; + let m: RegExpExecArray | null; + while ((m = NUMBER_RE.exec(text)) !== null) { + const n = normalizeBgNumber(m[0]); + if (n === null || !approxEq(n, target)) continue; + const before = text.slice(Math.max(0, m.index - 40), m.index); + if (excludeRe && excludeRe.test(before)) continue; + // A wider veto looks further back (~55 chars) for "в размер"/"ресурс"/increment markers that make an + // "…на " an amount, not a total. + if (wideVetoRe && wideVetoRe.test(text.slice(Math.max(0, m.index - 55), m.index))) continue; + if (!contextRe.test(before)) continue; + // #307 — a total needs a monetary marker bracketing the figure, else a bare "…на " matches a + // non-monetary number (days, article nos.) that coincidentally ≈ the target. A value keyword right + // before, OR a currency unit within the ~60 chars after, qualifies. + if (requireMoneyAnchor) { + const after = text.slice(m.index + m[0].length, m.index + m[0].length + 60); + // A non-monetary unit immediately after the figure (days/months/years/count/%) vetoes it before + // a downstream currency token can wrongly anchor it as money (#307). + if (NON_MONEY_UNIT_AFTER.test(after)) continue; + if (!MONEY_BEFORE.test(before) && !MONEY_AFTER.test(after)) continue; + } + return true; + } + return false; +} + +export function classifyAmendmentValue(input: AmendmentValueInput): AmendmentValueTreatment { + const b = input.valueBefore; + const a = input.valueAfter; + const d = input.valueDelta; + if (b === null || a === null || d === null || b <= 0 || a <= 0 || d <= 0) return { kind: 'none' }; + // Source self-consistency (a = b + d) is the precondition of the defect model. + if (!approxEq(a, b + d)) return { kind: 'none' }; + // A single annex whose "increment" is at least the whole prior value (2b ≤ a < 10b). Below 2b is a + // normal increase; ≥10b is a mis-key handled by #299's annex_suspect. + if (a < 2 * b || a >= 10 * b) return { kind: 'none' }; + + const text = input.texts.filter((t): t is string => !!t && t.trim() !== '').join(' '); + + // 1) The delta figure appears as an INCREMENT ("с ") — the value is genuinely correct, don't + // touch it. Checked FIRST so an exact 2× that the text calls a real increase is not mis-restated. + if (text && figureInContext(text, d, INCREMENT_CTX, null)) return { kind: 'genuine_increment' }; + + // 2) The delta figure appears as a TOTAL ("на ", "възлиза на …", "обща стойност … "). + // The true value_after is the delta (the announced new total). TOTAL_VETO rejects "в размер на"/ + // "ресурс"/increment phrasings that name the change amount, not the contract total. A monetary anchor + // is REQUIRED (#307) so a bare "…на " over a non-monetary number (days, article nos.) is rejected. + if (text && figureInContext(text, d, TOTAL_CTX, NOT_TOTAL_CTX, TOTAL_VETO, true)) { + return { kind: 'total_restated', correctedAfter: d }; + } + + // 3) Exact 2× (value_delta ≈ value_before): the "difference" field just echoed the OLD value, so + // value_after = before + before double-counts an UNCHANGED value (currency re-denomination or a + // non-value administrative annex). This restatement to value_before is only SAFE with a positive text + // signal (#307): ЗОП чл.116 ал.1 т.1 permits a genuine in-scope +100% via a pre-announced option/review + // clause that `outsideZop` cannot see, so a text-free rewrite could silently HALVE a legitimate value. + // Require either a "value unchanged / re-denomination" phrasing (RESTATE_UNCHANGED_CTX) or the + // before-value itself announced as the new total. Absent any signal, return `none` and let the + // arithmetic annex_total_suspect flag EXCLUDE the row (an honest gap beats a silent corruption). + // Still skipped for outside-ЗОП exception contracts, where a real +100% is legal. + if (approxEq(a, 2 * b) && !input.outsideZop && text) { + if ( + RESTATE_UNCHANGED_CTX.test(text) || + figureInContext(text, b, TOTAL_CTX, NOT_TOTAL_CTX, TOTAL_VETO, true) + ) { + return { kind: 'unchanged_restated', correctedAfter: b }; + } + } + + return { kind: 'none' }; +} + +// The single value the ETL needs: the corrected value_after when the text confirms a double-count, else +// null (leave value_after as the source gave it). +export function restatedValueAfter(input: AmendmentValueInput): number | null { + const t = classifyAmendmentValue(input); + return t.kind === 'total_restated' || t.kind === 'unchanged_restated' ? t.correctedAfter : null; +} + +export function isGenuineIncrement(input: AmendmentValueInput): boolean { + return classifyAmendmentValue(input).kind === 'genuine_increment'; +} + +// Convenience for the ETL staging: the treatment label to store on the raw amendment row (NULL when no +// signal), and the corrected value_after (NULL unless a double-count was confirmed). A non-null treatment +// tells derive/normalize NOT to arithmetic-flag the row (it is either corrected or confirmed-genuine). +export function amendmentValueTreatment(input: AmendmentValueInput): { + treatment: 'total_restated' | 'unchanged_restated' | 'genuine_increment' | null; + restatedAfter: number | null; +} { + const t = classifyAmendmentValue(input); + return { + treatment: t.kind === 'none' ? null : t.kind, + restatedAfter: + t.kind === 'total_restated' || t.kind === 'unchanged_restated' ? t.correctedAfter : null, + }; +} diff --git a/packages/ingest/src/base.test.ts b/packages/ingest/src/base.test.ts index d4d89b65..cb577d20 100644 --- a/packages/ingest/src/base.test.ts +++ b/packages/ingest/src/base.test.ts @@ -137,6 +137,54 @@ describe('base EOP mapper', () => { expect(baseSqlLiteral('annexes', 'value_delta', row?.value_delta)).toBe('-345.16'); }); + // #305 Tier-2: base.ts runs the validated основание-text heuristic (amendment-total.ts) for annexes and + // persists value_treatment + value_after_restated onto the raw row. A doubled value_after whose text + // announces the NEW TOTAL ("…на ") is restated to that true total; an untreated annex stays NULL. + it('populates value_treatment/value_after_restated for an annex whose text announces a new total', () => { + const row = mapBaseRecord( + 'annexes', + { + uniqueProcurementNumber: '00224-2025-0009', + contractNumber: '990001', + publicationDate: '05.03.2026', + lastContractValue: '442000', + currentContractValue: '981240', // doubled: source put the new TOTAL in the change field + contractValueDifference: '539240', + contractCurrency: 'BGN', + changeReason: 'Общата стойност на договора се променя на 539 240 лв.', + }, + { day: '2026-03-05', fetchedAt: '2026-03-05T00:00:00Z' }, + ); + + expect(row?.value_after).toBe(981240); // raw after left as the source gave it + expect(row?.value_treatment).toBe('total_restated'); + expect(row?.value_after_restated).toBe(539240); // the corrected true total + // The restated total must serialise as a bare number for the staging INSERT, not a quoted string. + expect(baseSqlLiteral('annexes', 'value_after_restated', row?.value_after_restated)).toBe( + '539240', + ); + }); + + it('leaves value_treatment/value_after_restated NULL for a >2× annex with no text total (not exact-2×)', () => { + // 2.5× (not exact 2×) and no announced total in the text ⇒ no confident signal ⇒ left to the flag. + const row = mapBaseRecord( + 'annexes', + { + uniqueProcurementNumber: '00224-2025-0010', + contractNumber: '990002', + publicationDate: '05.03.2026', + lastContractValue: '1000000', + currentContractValue: '2500000', + contractValueDifference: '1500000', + contractCurrency: 'BGN', + }, + { day: '2026-03-05', fetchedAt: '2026-03-05T00:00:00Z' }, + ); + + expect(row?.value_treatment).toBeNull(); + expect(row?.value_after_restated).toBeNull(); + }); + it('coerces signed reals without loosening the magnitude-only fields', () => { expect(toSignedReal('-345,16')).toBe(-345.16); expect(toSignedReal('-1 234,56')).toBe(-1234.56); diff --git a/packages/ingest/src/base.ts b/packages/ingest/src/base.ts index fea2b5ca..4020ee3b 100644 --- a/packages/ingest/src/base.ts +++ b/packages/ingest/src/base.ts @@ -1,5 +1,7 @@ // Base EOP plain-JSON adapter helpers. Pure and Worker-safe: no Node APIs. +import { amendmentValueTreatment } from './amendment-total.ts'; + export type BaseCategory = 'contracts' | 'tenders' | 'annexes'; export type BaseCoercionKind = | 'text' @@ -326,6 +328,10 @@ export const BASE_CATEGORIES: Record = { field('description', 'changeDescription', 'text'), field('reason', 'changeReason', 'text'), field('circumstances', 'changeReasonDescription', 'text'), + // #305 Tier-2 — computed from the основание text after the generic mapping (see mapBaseRecord), not + // read from a source key. key=null keeps the generic loop from touching them; mapBaseRecord sets them. + field('value_treatment', null, 'text'), + field('value_after_restated', null, 'real'), field('outside_zop', 'isExceptionContract', 'bool'), field('exemption_legal_basis', 'directAwardJustification', 'text'), field('correction_number', null, 'text'), @@ -379,9 +385,34 @@ export function mapBaseRecord( if (!cfg.keep(record)) return null; const row: BaseStagingRow = fixedValues(cat, meta); for (const f of cfg.fields) row[f.column] = f.key === null ? null : coerce(f.kind, record[f.key]); + // #305 Tier-2 — EOP annexes only: classify value_delta from the основание free text (the validated + // heuristic in amendment-total.ts) and persist the treatment label + corrected total onto the raw row. + // OCDS annexes never reach here (ocds.ts stages them, and they carry value_after = null anyway). + if (cat === 'annexes') { + const treatment = amendmentValueTreatment({ + valueBefore: numOrNull(row.value_before), + valueAfter: numOrNull(row.value_after), + valueDelta: numOrNull(row.value_delta), + currency: strOrNull(row.currency), + texts: [strOrNull(row.description), strOrNull(row.reason), strOrNull(row.circumstances)], + // #305 — outside-ЗОП exception contracts (isExceptionContract) are not bound by чл.116's +50% cap, + // so the text-free exact-2× restatement must not fire on them (see amendment-total.ts rule 3). + outsideZop: numOrNull(row.outside_zop) === 1, + }); + row.value_treatment = treatment.treatment; + row.value_after_restated = treatment.restatedAfter; + } return row; } +function numOrNull(v: BaseStagingValue | undefined): number | null { + return typeof v === 'number' ? v : null; +} + +function strOrNull(v: BaseStagingValue | undefined): string | null { + return typeof v === 'string' ? v : null; +} + // Hard ceiling on a single text literal's character length. EOP/registry text fields // (subjects, descriptions, names) are well under this; anything larger is corrupt or // hostile and is truncated rather than passed to sqlite (avoids SQLITE_TOOBIG / abuse of diff --git a/packages/ingest/tsconfig.json b/packages/ingest/tsconfig.json index b8ac7d61..9d4ac835 100644 --- a/packages/ingest/tsconfig.json +++ b/packages/ingest/tsconfig.json @@ -1,6 +1,8 @@ { "extends": "../../tsconfig.base.json", "compilerOptions": { + // base.ts imports './amendment-total.ts' with an explicit .ts extension (the ETL runtime consumes this + // source under plain Node). allowImportingTsExtensions is set in tsconfig.base.json for every consumer. "types": ["@cloudflare/workers-types"] }, "include": ["src"] diff --git a/scripts/derive-amendments.sql b/scripts/derive-amendments.sql index 23aaff87..23dcee7a 100644 --- a/scripts/derive-amendments.sql +++ b/scripts/derive-amendments.sql @@ -144,8 +144,10 @@ SET AND a.contract_number = raw_contracts.contract_number AND a.rn = 1 ), + -- #305 Tier-2: a text-confirmed double-count carries the corrected total in value_after_restated; use + -- it as the effective after so current_value reflects the true total, not the raw doubled value_after. current_value = ( - SELECT a.value_after FROM dedup a + SELECT COALESCE(a.value_after_restated, a.value_after) FROM dedup a WHERE a.unp = raw_contracts.unp AND a.contract_number = raw_contracts.contract_number AND a.value_after IS NOT NULL diff --git a/scripts/normalize-raw.sql b/scripts/normalize-raw.sql index 964132da..c5ba2c72 100644 --- a/scripts/normalize-raw.sql +++ b/scripts/normalize-raw.sql @@ -819,6 +819,7 @@ FROM ( CASE y.value_flag WHEN 'value_suspect' THEN y.proc_est_native WHEN 'annex_suspect' THEN COALESCE(y.signing_value, y.current_value) + WHEN 'annex_total_suspect' THEN COALESCE(y.signing_value, y.current_value) ELSE COALESCE(y.current_value, y.signing_value) END AS display_native, -- value_suspect is repaired directly from proc_est_eur in the outer amount_eur CASE; value_low and @@ -826,6 +827,7 @@ FROM ( CASE y.value_flag WHEN 'value_suspect' THEN NULL WHEN 'annex_suspect' THEN COALESCE(y.signing_value, y.current_value) + WHEN 'annex_total_suspect' THEN COALESCE(y.signing_value, y.current_value) ELSE COALESCE(y.current_value, y.signing_value) END AS trusted_native, -- Keep the companion currency paired with the exact native value chosen above. In particular, @@ -837,6 +839,10 @@ FROM ( WHEN y.signing_value IS NOT NULL THEN COALESCE(NULLIF(y.currency, ''), 'BGN') ELSE COALESCE(NULLIF(y.amendment_currency, ''), NULLIF(y.currency, ''), 'BGN') END + WHEN 'annex_total_suspect' THEN CASE + WHEN y.signing_value IS NOT NULL THEN COALESCE(NULLIF(y.currency, ''), 'BGN') + ELSE COALESCE(NULLIF(y.amendment_currency, ''), NULLIF(y.currency, ''), 'BGN') + END ELSE CASE WHEN y.current_value IS NOT NULL THEN COALESCE(NULLIF(y.amendment_currency, ''), NULLIF(y.currency, ''), 'BGN') ELSE COALESCE(NULLIF(y.currency, ''), 'BGN') @@ -943,6 +949,86 @@ FROM ( WHERE am.unp = c.unp AND am.contract_number = c.contract_number AND am.value_before > 0 AND am.value_after >= 10 * am.value_before ))))) THEN 'annex_suspect' + -- #305 value double-count: a driving annex reports a new TOTAL added to the old instead of + -- replacing it, so value_after ≈ 2× the OLD total. ЗОП чл.116 caps a single amendment at +50%, + -- so one step cannot legally more than double a contract — the ≥2× single step IS the defect + -- signal, wherever it sits in the chain. Scope: value_after in [2×,10×) a base that value_before + -- ties to a KNOWN prior total — signing_value OR a preceding annex's value_after (the multi-annex + -- case) — same currency. Slow legitimate climbs never reach ≥2× so stay untouched; the ≥10× + -- mis-key is #299's annex_suspect above; cross-currency doubles are an FX artefact ('review'); + -- and the ABS(... - current_value) tie binds this to the annex that DRIVES current_value, so a + -- doubled annex later superseded by a correct one is NOT flagged. + WHEN c.current_value IS NOT NULL AND c.signing_value > 0 AND EXISTS ( + SELECT 1 FROM raw_amendments am + WHERE am.unp = c.unp AND am.contract_number = c.contract_number + -- #305 Tier-2: a text-treated annex (restated total or confirmed-genuine increment) is NOT an + -- arithmetic suspect — value_treatment IS NOT NULL means the основание text already resolved it. + AND am.value_treatment IS NULL + -- #305 multi-annex: the doubled step need not be the FIRST annex. value_before may be a + -- prior cumulative total (a preceding annex's value_after), not signing. Anchor to signing + -- OR a legitimately-grown prior total (a prior annex that was itself not a double), while a + -- single ≥2× step (ЗОП чл.116 caps one amendment at +50%) is the defect wherever it sits. + AND am.value_before > 0 AND ( + ABS(am.value_before - c.signing_value) < 0.01 * c.signing_value + OR EXISTS ( + SELECT 1 FROM raw_amendments prev + WHERE prev.unp = am.unp AND prev.contract_number = am.contract_number + AND prev.value_after > 0 + AND ABS(prev.value_after - am.value_before) < 0.01 * am.value_before + -- ...and that prior total was itself reached legitimately (prev not a ≥2× double), + -- so a compounding chain where every step doubles is left untouched, not restated. + AND prev.value_before > 0 AND prev.value_after < 2 * prev.value_before + ) + -- #305 84818-class: an EXACT single-step 2× (value_after ≈ 2× value_before) on an ORPHAN + -- base — value_before ties neither signing NOR any prior annex's value_after (e.g. contract + -- 84818, whose annex base 76.77M is unrelated to the contract's values). A legal +100% in one + -- amendment is impossible (ЗОП чл.116), so flag → signing fallback (EXCLUDE); never REWRITES + -- (that stays gated, #307 HIGH-2). The orphan guard leaves a legitimate compounding-doubling + -- chain (each step's base ties the prior step) untouched, exactly as the legit-prior arm does. + OR ( + ABS(am.value_after - 2 * am.value_before) < 0.005 * am.value_before + AND NOT EXISTS ( + SELECT 1 FROM raw_amendments prev + WHERE prev.unp = am.unp AND prev.contract_number = am.contract_number + AND prev.value_after > 0 + AND ABS(prev.value_after - am.value_before) < 0.01 * am.value_before + ) + ) + ) + AND am.value_after >= 2 * am.value_before AND am.value_after < 10 * am.value_before + -- #305 M2: the double-count model presupposes a self-consistent row (value_after ≈ + -- value_before + value_delta). If value_delta is present and contradicts that, the model + -- provably does not apply — do NOT flag (mirrors the TS classifier's a≈b+d precondition). + AND (am.value_delta IS NULL + OR ABS(am.value_after - (am.value_before + am.value_delta)) < 0.01 * am.value_after) + AND ABS(am.value_after - c.current_value) < 0.01 + AND COALESCE(NULLIF(am.currency, ''), COALESCE(NULLIF(c.currency, ''), 'BGN')) + = COALESCE(NULLIF(c.currency, ''), 'BGN') + ) THEN 'annex_total_suspect' + -- #305 NEW-HIGH-1 (multi-annex chain contamination): the double-count correction is per-row and + -- does NOT propagate down a chain. When a PRIOR annex was double-count corrected (value_after was + -- doubled, restated down), a LATER annex still arrives from the feed computed on the CONTAMINATED + -- (raw, doubled) base. Its own step ratio is legitimate (<2×) so the gate above misses it, and the + -- prior annex is text-treated so it is excluded too — yet current_value inherited the doubled + -- total. Detect the driving annex whose value_before ties to a prior annex's RAW value_after where + -- that prior was restated to a lower total, and flag → signing fallback (an honest exclusion beats + -- a served overstatement) until per-chain value_before propagation (a follow-up) recomputes it. + WHEN c.current_value IS NOT NULL AND c.signing_value > 0 AND EXISTS ( + SELECT 1 FROM raw_amendments am + WHERE am.unp = c.unp AND am.contract_number = c.contract_number + AND am.value_treatment IS NULL + AND am.value_before > 0 + AND ABS(am.value_after - c.current_value) < 0.01 + AND EXISTS ( + SELECT 1 FROM raw_amendments prev + WHERE prev.unp = am.unp AND prev.contract_number = am.contract_number + AND prev.value_after_restated IS NOT NULL + AND prev.value_after_restated < prev.value_after + AND ABS(prev.value_after - am.value_before) < 0.01 * am.value_before + ) + AND COALESCE(NULLIF(am.currency, ''), COALESCE(NULLIF(c.currency, ''), 'BGN')) + = COALESCE(NULLIF(c.currency, ''), 'BGN') + ) THEN 'annex_total_suspect' WHEN c.proc_est_eur > 0 AND c.eff_eur >= 10 * c.proc_est_eur THEN 'review' ELSE 'ok' END AS value_flag, @@ -1283,6 +1369,86 @@ SELECT 1, WHERE am.unp = c.unp AND am.contract_number = c.contract_number AND am.value_before > 0 AND am.value_after >= 10 * am.value_before ))))) THEN 'annex_suspect' + -- #305 value double-count: a driving annex reports a new TOTAL added to the old instead of + -- replacing it, so value_after ≈ 2× the OLD total. ЗОП чл.116 caps a single amendment at +50%, + -- so one step cannot legally more than double a contract — the ≥2× single step IS the defect + -- signal, wherever it sits in the chain. Scope: value_after in [2×,10×) a base that value_before + -- ties to a KNOWN prior total — signing_value OR a preceding annex's value_after (the multi-annex + -- case) — same currency. Slow legitimate climbs never reach ≥2× so stay untouched; the ≥10× + -- mis-key is #299's annex_suspect above; cross-currency doubles are an FX artefact ('review'); + -- and the ABS(... - current_value) tie binds this to the annex that DRIVES current_value, so a + -- doubled annex later superseded by a correct one is NOT flagged. + WHEN c.current_value IS NOT NULL AND c.signing_value > 0 AND EXISTS ( + SELECT 1 FROM raw_amendments am + WHERE am.unp = c.unp AND am.contract_number = c.contract_number + -- #305 Tier-2: a text-treated annex (restated total or confirmed-genuine increment) is NOT an + -- arithmetic suspect — value_treatment IS NOT NULL means the основание text already resolved it. + AND am.value_treatment IS NULL + -- #305 multi-annex: the doubled step need not be the FIRST annex. value_before may be a + -- prior cumulative total (a preceding annex's value_after), not signing. Anchor to signing + -- OR a legitimately-grown prior total (a prior annex that was itself not a double), while a + -- single ≥2× step (ЗОП чл.116 caps one amendment at +50%) is the defect wherever it sits. + AND am.value_before > 0 AND ( + ABS(am.value_before - c.signing_value) < 0.01 * c.signing_value + OR EXISTS ( + SELECT 1 FROM raw_amendments prev + WHERE prev.unp = am.unp AND prev.contract_number = am.contract_number + AND prev.value_after > 0 + AND ABS(prev.value_after - am.value_before) < 0.01 * am.value_before + -- ...and that prior total was itself reached legitimately (prev not a ≥2× double), + -- so a compounding chain where every step doubles is left untouched, not restated. + AND prev.value_before > 0 AND prev.value_after < 2 * prev.value_before + ) + -- #305 84818-class: an EXACT single-step 2× (value_after ≈ 2× value_before) on an ORPHAN + -- base — value_before ties neither signing NOR any prior annex's value_after (e.g. contract + -- 84818, whose annex base 76.77M is unrelated to the contract's values). A legal +100% in one + -- amendment is impossible (ЗОП чл.116), so flag → signing fallback (EXCLUDE); never REWRITES + -- (that stays gated, #307 HIGH-2). The orphan guard leaves a legitimate compounding-doubling + -- chain (each step's base ties the prior step) untouched, exactly as the legit-prior arm does. + OR ( + ABS(am.value_after - 2 * am.value_before) < 0.005 * am.value_before + AND NOT EXISTS ( + SELECT 1 FROM raw_amendments prev + WHERE prev.unp = am.unp AND prev.contract_number = am.contract_number + AND prev.value_after > 0 + AND ABS(prev.value_after - am.value_before) < 0.01 * am.value_before + ) + ) + ) + AND am.value_after >= 2 * am.value_before AND am.value_after < 10 * am.value_before + -- #305 M2: the double-count model presupposes a self-consistent row (value_after ≈ + -- value_before + value_delta). If value_delta is present and contradicts that, the model + -- provably does not apply — do NOT flag (mirrors the TS classifier's a≈b+d precondition). + AND (am.value_delta IS NULL + OR ABS(am.value_after - (am.value_before + am.value_delta)) < 0.01 * am.value_after) + AND ABS(am.value_after - c.current_value) < 0.01 + AND COALESCE(NULLIF(am.currency, ''), COALESCE(NULLIF(c.currency, ''), 'BGN')) + = COALESCE(NULLIF(c.currency, ''), 'BGN') + ) THEN 'annex_total_suspect' + -- #305 NEW-HIGH-1 (multi-annex chain contamination): the double-count correction is per-row and + -- does NOT propagate down a chain. When a PRIOR annex was double-count corrected (value_after was + -- doubled, restated down), a LATER annex still arrives from the feed computed on the CONTAMINATED + -- (raw, doubled) base. Its own step ratio is legitimate (<2×) so the gate above misses it, and the + -- prior annex is text-treated so it is excluded too — yet current_value inherited the doubled + -- total. Detect the driving annex whose value_before ties to a prior annex's RAW value_after where + -- that prior was restated to a lower total, and flag → signing fallback (an honest exclusion beats + -- a served overstatement) until per-chain value_before propagation (a follow-up) recomputes it. + WHEN c.current_value IS NOT NULL AND c.signing_value > 0 AND EXISTS ( + SELECT 1 FROM raw_amendments am + WHERE am.unp = c.unp AND am.contract_number = c.contract_number + AND am.value_treatment IS NULL + AND am.value_before > 0 + AND ABS(am.value_after - c.current_value) < 0.01 + AND EXISTS ( + SELECT 1 FROM raw_amendments prev + WHERE prev.unp = am.unp AND prev.contract_number = am.contract_number + AND prev.value_after_restated IS NOT NULL + AND prev.value_after_restated < prev.value_after + AND ABS(prev.value_after - am.value_before) < 0.01 * am.value_before + ) + AND COALESCE(NULLIF(am.currency, ''), COALESCE(NULLIF(c.currency, ''), 'BGN')) + = COALESCE(NULLIF(c.currency, ''), 'BGN') + ) THEN 'annex_total_suspect' WHEN c.proc_est_eur > 0 AND c.eff_eur >= 10 * c.proc_est_eur THEN 'review' ELSE 'ok' END AS value_flag, @@ -1349,6 +1515,7 @@ SELECT 1, ) c WHERE CASE c.value_flag WHEN 'annex_suspect' THEN COALESCE(c.signing_value, c.current_value) + WHEN 'annex_total_suspect' THEN COALESCE(c.signing_value, c.current_value) ELSE COALESCE(c.current_value, c.signing_value) END IS NOT NULL AND EXISTS (SELECT 1 FROM tenders te WHERE te.id = 't:' || c.unp) @@ -1376,6 +1543,7 @@ SELECT (SELECT contract_candidates FROM pipeline_stats) AS contract_candidates, (SELECT COUNT(*) FROM contracts WHERE value_flag = 'value_suspect') AS value_suspect, (SELECT COUNT(*) FROM contracts WHERE value_flag = 'annex_suspect') AS annex_suspect, + (SELECT COUNT(*) FROM contracts WHERE value_flag = 'annex_total_suspect') AS annex_total_suspect, (SELECT COUNT(*) FROM contracts WHERE value_flag = 'review') AS review, (SELECT COUNT(*) FROM contracts WHERE fx_converted = 1) AS fx_converted, (SELECT ROUND(SUM(amount_eur) / 1e9, 2) FROM contracts) AS clean_total_eur_bn, diff --git a/scripts/precompute.sql b/scripts/precompute.sql index c79a26ef..f57cbedd 100644 --- a/scripts/precompute.sql +++ b/scripts/precompute.sql @@ -22,8 +22,9 @@ -- signing/current in EUR for the contract page's estimated→signing→current strip. -- BGN at the fixed peg (÷1.95583), EUR as-is, foreign at the row's stored fx_rate (eur_per_unit). -- Display rule: NULL where the figure is suspect, so the caller renders „данните се преглеждат", --- never a fabricated number. signing suppressed for value_suspect; current suppressed for value_ or --- annex_suspect (the suspect annex is the bad part). estimated_value_eur is derived per-request on +-- never a fabricated number. signing suppressed for value_suspect; current suppressed for value_, +-- annex_suspect or annex_total_suspect (#305; the suspect annex is the bad part). estimated_value_eur +-- is derived per-request on -- the contract detail loader from the tender (procurement-level, shared across a multi-lot prepiska). UPDATE contracts SET signing_value_eur = CASE @@ -33,7 +34,7 @@ UPDATE contracts SET WHEN fx_rate IS NOT NULL THEN signing_value * fx_rate ELSE NULL END, current_value_eur = CASE - WHEN value_flag IN ('value_suspect','annex_suspect') OR current_value IS NULL THEN NULL + WHEN value_flag IN ('value_suspect','annex_suspect','annex_total_suspect') OR current_value IS NULL THEN NULL WHEN COALESCE(NULLIF(current_value_currency, ''), NULLIF(currency, ''), 'BGN') = 'EUR' THEN current_value WHEN COALESCE(NULLIF(current_value_currency, ''), NULLIF(currency, ''), 'BGN') = 'BGN' THEN current_value / 1.95583 WHEN fx_rate IS NOT NULL THEN current_value * fx_rate diff --git a/scripts/promote-amendments.sql b/scripts/promote-amendments.sql index 5d72e367..b7822315 100644 --- a/scripts/promote-amendments.sql +++ b/scripts/promote-amendments.sql @@ -8,7 +8,7 @@ DELETE FROM amendments; INSERT OR REPLACE INTO amendments ( id, natural_key, contract_number, unp, value_before, value_after, value_delta, currency, - published_at, document_number, description, source + published_at, document_number, description, source, value_restated, value_treatment, value_suspect ) WITH keyed AS ( SELECT @@ -40,13 +40,62 @@ SELECT contract_number, unp, value_before, - value_after, - value_delta, + -- #305 Tier-2: serve the effective (text-corrected) after and a self-consistent delta; a restated + -- annex carries the true total, an untreated one is unchanged. + COALESCE(value_after_restated, value_after), + COALESCE(value_after_restated, value_after) - value_before, currency, published_at, document_number, description, - source + source, + CASE WHEN value_after_restated IS NOT NULL THEN 1 ELSE 0 END, + value_treatment, + -- #305 residual: mark a suspected double-count that is NOT already text-treated so the UI suppresses + -- the untrusted value_after. Mirrors normalize-raw.sql's annex_total_suspect arithmetic gate, but + -- joined to raw_contracts for the contract's signing_value/currency (this served INSERT has no + -- contract row to read). value_treatment IS NULL keeps a restated/genuine row out (value_restated + -- already owns those). No current_value tie here: the tie in normalize-raw only decides whether the + -- CONTRACT is flagged; the per-row marker suppresses any row whose after is an unbridgeable double. + CASE WHEN value_treatment IS NULL + AND value_before > 0 + AND value_after >= 2 * value_before AND value_after < 10 * value_before + -- #305 M2 self-consistency: skip when value_delta is present and a ≉ b + d (model N/A). + AND (value_delta IS NULL OR ABS(value_after - (value_before + value_delta)) < 0.01 * value_after) + AND EXISTS ( + SELECT 1 FROM raw_contracts rc + WHERE rc.unp = dedup.unp AND rc.contract_number = dedup.contract_number + AND rc.signing_value > 0 + -- #305 multi-annex: value_before may be a prior cumulative total (a preceding annex's + -- value_after), not signing. Anchor to signing OR a legitimately-grown prior total (prev not + -- itself a double); a single ≥2× step violates ЗОП чл.116 wherever it sits (see normalize-raw.sql). + AND ( + ABS(dedup.value_before - rc.signing_value) < 0.01 * rc.signing_value + OR EXISTS ( + SELECT 1 FROM raw_amendments prev + WHERE prev.unp = dedup.unp AND prev.contract_number = dedup.contract_number + AND prev.value_after > 0 + AND ABS(prev.value_after - dedup.value_before) < 0.01 * dedup.value_before + -- ...and that prior total was itself reached legitimately (prev not a ≥2× double). + AND prev.value_before > 0 AND prev.value_after < 2 * prev.value_before + ) + -- #305 84818-class: EXACT single-step 2× on an ORPHAN base (value_before ties neither signing + -- nor any prior annex) — mark the row suspect; never rewrites (see normalize-raw.sql). The + -- orphan guard leaves compounding chains untouched. + OR ( + ABS(dedup.value_after - 2 * dedup.value_before) < 0.005 * dedup.value_before + AND NOT EXISTS ( + SELECT 1 FROM raw_amendments prev + WHERE prev.unp = dedup.unp AND prev.contract_number = dedup.contract_number + AND prev.value_after > 0 + AND ABS(prev.value_after - dedup.value_before) < 0.01 * dedup.value_before + ) + ) + ) + AND COALESCE(NULLIF(dedup.currency, ''), COALESCE(NULLIF(rc.currency, ''), 'BGN')) + = COALESCE(NULLIF(rc.currency, ''), 'BGN') + ) + THEN 1 ELSE 0 END FROM dedup WHERE rn = 1; diff --git a/scripts/refresh-slice.sql b/scripts/refresh-slice.sql index d74ab266..cdaf8527 100644 --- a/scripts/refresh-slice.sql +++ b/scripts/refresh-slice.sql @@ -1063,7 +1063,7 @@ FROM ( ELSE q.signing_value * q.fx_rate END AS signing_value_eur, CASE - WHEN q.value_flag IN ('value_suspect', 'annex_suspect') OR q.current_value IS NULL THEN NULL + WHEN q.value_flag IN ('value_suspect', 'annex_suspect', 'annex_total_suspect') OR q.current_value IS NULL THEN NULL WHEN q.current_value_currency = 'EUR' THEN q.current_value WHEN q.current_value_currency = 'BGN' THEN q.current_value / 1.95583 ELSE q.current_value * q.fx_rate @@ -1073,11 +1073,13 @@ FROM ( CASE y.value_flag WHEN 'value_suspect' THEN y.proc_est_native WHEN 'annex_suspect' THEN COALESCE(y.signing_value, y.current_value) + WHEN 'annex_total_suspect' THEN COALESCE(y.signing_value, y.current_value) ELSE COALESCE(y.current_value, y.signing_value) END AS display_native, CASE y.value_flag WHEN 'value_suspect' THEN NULL WHEN 'annex_suspect' THEN COALESCE(y.signing_value, y.current_value) + WHEN 'annex_total_suspect' THEN COALESCE(y.signing_value, y.current_value) ELSE COALESCE(y.current_value, y.signing_value) END AS trusted_native, CASE y.value_flag @@ -1087,6 +1089,11 @@ FROM ( ELSE COALESCE((SELECT NULLIF(w.currency, '') FROM refresh_amendment_winners w WHERE w.unp = y.unp AND w.contract_number = y.contract_number), NULLIF(y.currency, ''), 'BGN') END + WHEN 'annex_total_suspect' THEN CASE + WHEN y.signing_value IS NOT NULL THEN COALESCE(NULLIF(y.currency, ''), 'BGN') + ELSE COALESCE((SELECT NULLIF(w.currency, '') FROM refresh_amendment_winners w + WHERE w.unp = y.unp AND w.contract_number = y.contract_number), NULLIF(y.currency, ''), 'BGN') + END ELSE CASE WHEN y.current_value IS NOT NULL THEN COALESCE((SELECT NULLIF(w.currency, '') FROM refresh_amendment_winners w WHERE w.unp = y.unp AND w.contract_number = y.contract_number), NULLIF(y.currency, ''), 'BGN') @@ -1182,6 +1189,55 @@ FROM ( WHERE am.unp = c.unp AND am.contract_number = c.contract_number AND am.value_before > 0 AND am.value_after >= 10 * am.value_before ))))) THEN 'annex_suspect' + -- #305 value double-count: a driving annex reports a new TOTAL added to the old instead of + -- replacing it, so value_after ≈ 2× the OLD total. ЗОП чл.116 caps a single amendment at +50%, + -- so one step cannot legally more than double a contract — the ≥2× single step IS the defect + -- signal, wherever it sits in the chain. Scope: value_after in [2×,10×) a base that value_before + -- ties to a KNOWN prior total — signing_value OR a preceding annex's value_after (the multi-annex + -- case) — same currency. Slow legitimate climbs never reach ≥2× so stay untouched; the ≥10× + -- mis-key is #299's annex_suspect above; cross-currency doubles are an FX artefact ('review'); + -- and the ABS(... - current_value) tie binds this to the annex that DRIVES current_value, so a + -- doubled annex later superseded by a correct one is NOT flagged. + WHEN c.current_value IS NOT NULL AND c.signing_value > 0 AND EXISTS ( + SELECT 1 FROM raw_amendments am + WHERE am.unp = c.unp AND am.contract_number = c.contract_number + -- #305 Tier-2: skip text-treated annexes (restated total or confirmed-genuine increment). + AND am.value_treatment IS NULL + -- #305 multi-annex: value_before may be a prior cumulative total (a preceding annex's + -- value_after), not signing. Anchor to signing OR a legitimately-grown prior total (prev + -- not itself a double); a single ≥2× step violates ЗОП чл.116 wherever it sits (see normalize-raw.sql). + AND am.value_before > 0 AND ( + ABS(am.value_before - c.signing_value) < 0.01 * c.signing_value + OR EXISTS ( + SELECT 1 FROM raw_amendments prev + WHERE prev.unp = am.unp AND prev.contract_number = am.contract_number + AND prev.value_after > 0 + AND ABS(prev.value_after - am.value_before) < 0.01 * am.value_before + -- ...and that prior total was itself reached legitimately (prev not a ≥2× double), + -- so a compounding chain where every step doubles is left untouched, not restated. + AND prev.value_before > 0 AND prev.value_after < 2 * prev.value_before + ) + -- #305 84818-class: an EXACT single-step 2× on an ORPHAN base (value_before ties neither + -- signing nor any prior annex) is the ЗОП чл.116 defect signature — flag (→ signing + -- fallback, EXCLUDE); never rewrites. The orphan guard leaves compounding chains untouched + -- (see normalize-raw.sql). + OR ( + ABS(am.value_after - 2 * am.value_before) < 0.005 * am.value_before + AND NOT EXISTS ( + SELECT 1 FROM raw_amendments prev + WHERE prev.unp = am.unp AND prev.contract_number = am.contract_number + AND prev.value_after > 0 + AND ABS(prev.value_after - am.value_before) < 0.01 * am.value_before + ) + ) + ) + AND am.value_after >= 2 * am.value_before AND am.value_after < 10 * am.value_before + -- #305 M2 self-consistency: skip when value_delta is present and a ≉ b + d (model N/A). + AND (am.value_delta IS NULL OR ABS(am.value_after - (am.value_before + am.value_delta)) < 0.01 * am.value_after) + AND ABS(am.value_after - c.current_value) < 0.01 + AND COALESCE(NULLIF(am.currency, ''), COALESCE(NULLIF(c.currency, ''), 'BGN')) + = COALESCE(NULLIF(c.currency, ''), 'BGN') + ) THEN 'annex_total_suspect' WHEN c.proc_est_eur > 0 AND c.eff_eur >= 10 * c.proc_est_eur THEN 'review' ELSE 'ok' END AS value_flag, @@ -1351,7 +1407,7 @@ FROM ( ELSE q.signing_value * q.fx_rate END AS signing_value_eur, CASE - WHEN q.value_flag IN ('value_suspect', 'annex_suspect') OR q.current_value IS NULL THEN NULL + WHEN q.value_flag IN ('value_suspect', 'annex_suspect', 'annex_total_suspect') OR q.current_value IS NULL THEN NULL WHEN q.current_value_currency = 'EUR' THEN q.current_value WHEN q.current_value_currency = 'BGN' THEN q.current_value / 1.95583 ELSE q.current_value * q.fx_rate @@ -1361,11 +1417,13 @@ FROM ( CASE y.value_flag WHEN 'value_suspect' THEN y.proc_est_native WHEN 'annex_suspect' THEN COALESCE(y.signing_value, y.current_value) + WHEN 'annex_total_suspect' THEN COALESCE(y.signing_value, y.current_value) ELSE COALESCE(y.current_value, y.signing_value) END AS display_native, CASE y.value_flag WHEN 'value_suspect' THEN NULL WHEN 'annex_suspect' THEN COALESCE(y.signing_value, y.current_value) + WHEN 'annex_total_suspect' THEN COALESCE(y.signing_value, y.current_value) ELSE COALESCE(y.current_value, y.signing_value) END AS trusted_native, CASE y.value_flag @@ -1375,6 +1433,11 @@ FROM ( ELSE COALESCE((SELECT NULLIF(w.currency, '') FROM refresh_amendment_winners w WHERE w.unp = y.unp AND w.contract_number = y.contract_number), NULLIF(y.currency, ''), 'BGN') END + WHEN 'annex_total_suspect' THEN CASE + WHEN y.signing_value IS NOT NULL THEN COALESCE(NULLIF(y.currency, ''), 'BGN') + ELSE COALESCE((SELECT NULLIF(w.currency, '') FROM refresh_amendment_winners w + WHERE w.unp = y.unp AND w.contract_number = y.contract_number), NULLIF(y.currency, ''), 'BGN') + END ELSE CASE WHEN y.current_value IS NOT NULL THEN COALESCE((SELECT NULLIF(w.currency, '') FROM refresh_amendment_winners w WHERE w.unp = y.unp AND w.contract_number = y.contract_number), NULLIF(y.currency, ''), 'BGN') @@ -1474,6 +1537,55 @@ FROM ( WHERE am.unp = c.unp AND am.contract_number = c.contract_number AND am.value_before > 0 AND am.value_after >= 10 * am.value_before ))))) THEN 'annex_suspect' + -- #305 value double-count: a driving annex reports a new TOTAL added to the old instead of + -- replacing it, so value_after ≈ 2× the OLD total. ЗОП чл.116 caps a single amendment at +50%, + -- so one step cannot legally more than double a contract — the ≥2× single step IS the defect + -- signal, wherever it sits in the chain. Scope: value_after in [2×,10×) a base that value_before + -- ties to a KNOWN prior total — signing_value OR a preceding annex's value_after (the multi-annex + -- case) — same currency. Slow legitimate climbs never reach ≥2× so stay untouched; the ≥10× + -- mis-key is #299's annex_suspect above; cross-currency doubles are an FX artefact ('review'); + -- and the ABS(... - current_value) tie binds this to the annex that DRIVES current_value, so a + -- doubled annex later superseded by a correct one is NOT flagged. + WHEN c.current_value IS NOT NULL AND c.signing_value > 0 AND EXISTS ( + SELECT 1 FROM raw_amendments am + WHERE am.unp = c.unp AND am.contract_number = c.contract_number + -- #305 Tier-2: skip text-treated annexes (restated total or confirmed-genuine increment). + AND am.value_treatment IS NULL + -- #305 multi-annex: value_before may be a prior cumulative total (a preceding annex's + -- value_after), not signing. Anchor to signing OR a legitimately-grown prior total (prev + -- not itself a double); a single ≥2× step violates ЗОП чл.116 wherever it sits (see normalize-raw.sql). + AND am.value_before > 0 AND ( + ABS(am.value_before - c.signing_value) < 0.01 * c.signing_value + OR EXISTS ( + SELECT 1 FROM raw_amendments prev + WHERE prev.unp = am.unp AND prev.contract_number = am.contract_number + AND prev.value_after > 0 + AND ABS(prev.value_after - am.value_before) < 0.01 * am.value_before + -- ...and that prior total was itself reached legitimately (prev not a ≥2× double), + -- so a compounding chain where every step doubles is left untouched, not restated. + AND prev.value_before > 0 AND prev.value_after < 2 * prev.value_before + ) + -- #305 84818-class: an EXACT single-step 2× on an ORPHAN base (value_before ties neither + -- signing nor any prior annex) is the ЗОП чл.116 defect signature — flag (→ signing + -- fallback, EXCLUDE); never rewrites. The orphan guard leaves compounding chains untouched + -- (see normalize-raw.sql). + OR ( + ABS(am.value_after - 2 * am.value_before) < 0.005 * am.value_before + AND NOT EXISTS ( + SELECT 1 FROM raw_amendments prev + WHERE prev.unp = am.unp AND prev.contract_number = am.contract_number + AND prev.value_after > 0 + AND ABS(prev.value_after - am.value_before) < 0.01 * am.value_before + ) + ) + ) + AND am.value_after >= 2 * am.value_before AND am.value_after < 10 * am.value_before + -- #305 M2 self-consistency: skip when value_delta is present and a ≉ b + d (model N/A). + AND (am.value_delta IS NULL OR ABS(am.value_after - (am.value_before + am.value_delta)) < 0.01 * am.value_after) + AND ABS(am.value_after - c.current_value) < 0.01 + AND COALESCE(NULLIF(am.currency, ''), COALESCE(NULLIF(c.currency, ''), 'BGN')) + = COALESCE(NULLIF(c.currency, ''), 'BGN') + ) THEN 'annex_total_suspect' WHEN c.proc_est_eur > 0 AND c.eff_eur >= 10 * c.proc_est_eur THEN 'review' ELSE 'ok' END AS value_flag, @@ -1605,7 +1717,7 @@ WHERE source LIKE 'ocds:%' INSERT OR REPLACE INTO amendments ( id, natural_key, contract_number, unp, value_before, value_after, value_delta, currency, - published_at, document_number, description, source + published_at, document_number, description, source, value_restated, value_treatment, value_suspect ) WITH keyed AS ( SELECT @@ -1637,13 +1749,62 @@ SELECT contract_number, unp, value_before, - value_after, - value_delta, + -- #305 Tier-2: serve the effective (text-corrected) after and a self-consistent delta; the current_value + -- rollup below reads this served value_after, so a restated annex drives current_value with the true total. + COALESCE(value_after_restated, value_after), + COALESCE(value_after_restated, value_after) - value_before, currency, published_at, document_number, description, - source + source, + CASE WHEN value_after_restated IS NOT NULL THEN 1 ELSE 0 END, + value_treatment, + -- #305 residual: mark a suspected double-count that is NOT already text-treated so the UI suppresses + -- the untrusted value_after. Mirrors normalize-raw.sql's annex_total_suspect arithmetic gate, but + -- joined to raw_contracts for the contract's signing_value/currency (this served INSERT has no + -- contract row to read). value_treatment IS NULL keeps a restated/genuine row out (value_restated + -- already owns those). No current_value tie here: the tie in normalize-raw only decides whether the + -- CONTRACT is flagged; the per-row marker suppresses any row whose after is an unbridgeable double. + CASE WHEN value_treatment IS NULL + AND value_before > 0 + AND value_after >= 2 * value_before AND value_after < 10 * value_before + -- #305 M2 self-consistency: skip when value_delta is present and a ≉ b + d (model N/A). + AND (value_delta IS NULL OR ABS(value_after - (value_before + value_delta)) < 0.01 * value_after) + AND EXISTS ( + SELECT 1 FROM raw_contracts rc + WHERE rc.unp = dedup.unp AND rc.contract_number = dedup.contract_number + AND rc.signing_value > 0 + -- #305 multi-annex: value_before may be a prior cumulative total (a preceding annex's + -- value_after), not signing. Anchor to signing OR a legitimately-grown prior total (prev not + -- itself a double); a single ≥2× step violates ЗОП чл.116 wherever it sits (see normalize-raw.sql). + AND ( + ABS(dedup.value_before - rc.signing_value) < 0.01 * rc.signing_value + OR EXISTS ( + SELECT 1 FROM raw_amendments prev + WHERE prev.unp = dedup.unp AND prev.contract_number = dedup.contract_number + AND prev.value_after > 0 + AND ABS(prev.value_after - dedup.value_before) < 0.01 * dedup.value_before + -- ...and that prior total was itself reached legitimately (prev not a ≥2× double). + AND prev.value_before > 0 AND prev.value_after < 2 * prev.value_before + ) + -- #305 84818-class: EXACT single-step 2× on an ORPHAN base (value_before ties neither signing + -- nor any prior annex) — mark the row suspect; never rewrites (see normalize-raw.sql). The + -- orphan guard leaves compounding chains untouched. + OR ( + ABS(dedup.value_after - 2 * dedup.value_before) < 0.005 * dedup.value_before + AND NOT EXISTS ( + SELECT 1 FROM raw_amendments prev + WHERE prev.unp = dedup.unp AND prev.contract_number = dedup.contract_number + AND prev.value_after > 0 + AND ABS(prev.value_after - dedup.value_before) < 0.01 * dedup.value_before + ) + ) + ) + AND COALESCE(NULLIF(dedup.currency, ''), COALESCE(NULLIF(rc.currency, ''), 'BGN')) + = COALESCE(NULLIF(rc.currency, ''), 'BGN') + ) + THEN 1 ELSE 0 END FROM dedup WHERE rn = 1; @@ -1659,7 +1820,10 @@ SET WHERE a.unp = substr(contracts.tender_id, 3) AND a.contract_number = contracts.contract_number AND a.value_after IS NOT NULL - ORDER BY a.published_at DESC, a.id DESC + -- #305: tie-break on natural_key to match the full-rebuild path (derive-amendments.sql), so the + -- driving amendment picked for current_value is identical across full and slice when two annexes + -- share published_at. `id` (row insertion order) diverged from natural_key and could pick a different row. + ORDER BY a.published_at DESC, a.natural_key DESC LIMIT 1 ), current_value_currency = ( @@ -1667,7 +1831,9 @@ SET WHERE a.unp = substr(contracts.tender_id, 3) AND a.contract_number = contracts.contract_number AND a.value_after IS NOT NULL - ORDER BY a.published_at DESC, a.id DESC + -- #305: same natural_key tie-break as current_value above, so the currency comes from the same + -- driving amendment the value does. + ORDER BY a.published_at DESC, a.natural_key DESC LIMIT 1 ) WHERE (id GLOB 'c:[eo]:*' AND EXISTS ( @@ -1734,7 +1900,90 @@ WITH contract_base AS ( WHERE am.unp = substr(c.tender_id, 3) AND am.contract_number = c.contract_number AND am.value_before > 0 AND am.value_after >= 10 * am.value_before - ) AS has_step10 + ) AS has_step10, + -- #305 value double-count: a driving annex whose value_before ≈ a KNOWN prior total — the contract's + -- signing_value OR a preceding annex's value_after (the multi-annex case) — with value_after in + -- [2×,10×) that base, same currency, and matching current_value. Checked against the CUMULATIVE domain + -- amendments, matching where this pass re-rolls current_value from. ЗОП чл.116 caps a single amendment + -- at +50%, so the ≥2× step is the defect signal wherever it sits; slow climbs never reach ≥2×, and the + -- ≥10× mis-key and cross-currency cases are handled elsewhere. + EXISTS ( + SELECT 1 FROM amendments am + WHERE am.unp = substr(c.tender_id, 3) + AND am.contract_number = c.contract_number + -- #305 Tier-2: skip text-treated annexes. Restated totals already stop matching (served value_after + -- is the corrected total, no longer ≈2× before), but the explicit guard also covers confirmed-genuine + -- increments, whose value_after is legitimately ≥2× and must NOT be arithmetic-flagged. + AND am.value_treatment IS NULL + AND am.value_before > 0 AND c.signing_value > 0 + -- #305 multi-annex: value_before may be a prior cumulative total (a preceding annex's + -- value_after), not signing — anchor to signing OR a legitimately-grown prior total, prev not + -- itself a double (see normalize-raw.sql). + AND ( + ABS(am.value_before - c.signing_value) < 0.01 * c.signing_value + OR EXISTS ( + SELECT 1 FROM amendments prev + WHERE prev.unp = am.unp AND prev.contract_number = am.contract_number + AND prev.value_after > 0 + -- #305 NEW-HIGH-2: this reconciliation reads the CUMULATIVE served `amendments` (a prior-window + -- annex is not in this window's raw_amendments), but the full path anchors on RAW values. For a + -- #305-restated prev the served value_after is the CORRECTED (lower) total, not the raw one, so + -- `value_after < 2*value_before` flips true and the gate would disagree with the full rebuild + -- (flag flips between the daily slice and the next full derive). Restrict the anchor to + -- non-restated prevs, whose served value_after == raw value_after — reproducing the full-path + -- (raw) decision without losing cross-window history. + AND prev.value_restated = 0 + AND ABS(prev.value_after - am.value_before) < 0.01 * am.value_before + -- ...and that prior total was itself reached legitimately (prev not a ≥2× double), + -- so a compounding chain where every step doubles is left untouched, not restated. + AND prev.value_before > 0 AND prev.value_after < 2 * prev.value_before + ) + -- #305 84818-class: an EXACT single-step 2× on an ORPHAN base (value_before ties neither signing + -- nor any prior served annex) is the ЗОП чл.116 defect signature — flag (→ signing fallback, + -- EXCLUDE); never rewrites. The orphan guard leaves compounding chains untouched (see + -- normalize-raw.sql). + OR ( + ABS(am.value_after - 2 * am.value_before) < 0.005 * am.value_before + AND NOT EXISTS ( + SELECT 1 FROM amendments prev + WHERE prev.unp = am.unp AND prev.contract_number = am.contract_number + AND prev.value_after > 0 + AND ABS(prev.value_after - am.value_before) < 0.01 * am.value_before + ) + ) + ) + AND am.value_after >= 2 * am.value_before AND am.value_after < 10 * am.value_before + -- #305 M2 self-consistency: skip when value_delta is present and a ≉ b + d (model N/A). + AND (am.value_delta IS NULL OR ABS(am.value_after - (am.value_before + am.value_delta)) < 0.01 * am.value_after) + AND ABS(am.value_after - c.current_value) < 0.01 + AND COALESCE(NULLIF(am.currency, ''), COALESCE(NULLIF(c.currency, ''), 'BGN')) + = COALESCE(NULLIF(c.currency, ''), 'BGN') + ) AS has_double, + -- #305 NEW-HIGH-1 (multi-annex chain contamination), slice mirror. The full path (normalize-raw.sql) + -- detects this on RAW values (prev.value_after_restated < prev.value_after AND am.value_before ≈ raw + -- prev.value_after). The slice reads the CUMULATIVE served `amendments`, which does NOT retain the raw + -- value_after of a restated prev — so this is a CONSERVATIVE approximation: a driving annex whose + -- value_before sits ABOVE a restated prior annex's CORRECTED total (it rode the raw, doubled base) but + -- within a contamination band (< 2× the corrected total, i.e. not a fresh legitimate double). Flags → + -- signing fallback, matching the full path's honest exclusion. Exactness is restored on the next full + -- rebuild; a follow-up value_before-propagation PR removes the approximation entirely. + EXISTS ( + SELECT 1 FROM amendments am + WHERE am.unp = substr(c.tender_id, 3) + AND am.contract_number = c.contract_number + AND am.value_treatment IS NULL + AND am.value_before > 0 + AND ABS(am.value_after - c.current_value) < 0.01 + AND EXISTS ( + SELECT 1 FROM amendments prev + WHERE prev.unp = am.unp AND prev.contract_number = am.contract_number + AND prev.value_restated = 1 + AND am.value_before > prev.value_after + AND am.value_before < 2 * prev.value_after + ) + AND COALESCE(NULLIF(am.currency, ''), COALESCE(NULLIF(c.currency, ''), 'BGN')) + = COALESCE(NULLIF(c.currency, ''), 'BGN') + ) AS has_contaminated_base FROM contracts c JOIN tenders te ON te.id = c.tender_id WHERE ( @@ -1757,9 +2006,10 @@ WITH contract_base AS ( ), base AS ( SELECT id, currency, signing_value, current_value, current_value_currency, fx_rate, proc_est_eur, proc_est_native, CASE - WHEN c.value_flag <> 'annex_suspect' + WHEN c.value_flag NOT IN ('annex_suspect', 'annex_total_suspect') AND NOT (c.current_value IS NOT NULL AND (c.current_value < 0 OR (c.signing_value > 0 AND (c.current_value / c.signing_value >= 100 OR (c.current_value / c.signing_value >= 5 AND c.has_step10))))) + AND NOT (c.current_value IS NOT NULL AND c.signing_value > 0 AND (c.has_double OR c.has_contaminated_base)) THEN c.value_flag WHEN c.eff_eur > 2000000000 OR (c.proc_est_eur >= 1000 AND (c.eff_eur > 200 * c.proc_est_eur -- Dropped decimal point: the value was entered in стотинки, so it lands at almost exactly @@ -1771,6 +2021,11 @@ WITH contract_base AS ( -- The step alone is NOT enough — some chains have a huge step that a later annex pulls -- back below signing, and flagging those would RAISE the shown value, not repair it. OR (c.current_value / c.signing_value >= 5 AND c.has_step10)))) THEN 'annex_suspect' + -- #305 single-annex value double-count: the driving annex more than doubled the contract in one + -- step (ЗОП чл.116 caps a single amendment at +50%). has_double already ties to current_value. + -- has_contaminated_base additionally catches a legitimate-looking later annex riding a doubled base + -- (#305 NEW-HIGH-1) — both fall back to signing. + WHEN c.current_value IS NOT NULL AND c.signing_value > 0 AND (c.has_double OR c.has_contaminated_base) THEN 'annex_total_suspect' WHEN c.proc_est_eur > 0 AND c.eff_eur >= 10 * c.proc_est_eur THEN 'review' ELSE 'ok' END AS new_value_flag @@ -1780,11 +2035,13 @@ WITH contract_base AS ( CASE new_value_flag WHEN 'value_suspect' THEN proc_est_native WHEN 'annex_suspect' THEN COALESCE(signing_value, current_value) + WHEN 'annex_total_suspect' THEN COALESCE(signing_value, current_value) ELSE COALESCE(current_value, signing_value) END AS display_native, CASE new_value_flag WHEN 'value_suspect' THEN NULL WHEN 'annex_suspect' THEN COALESCE(signing_value, current_value) + WHEN 'annex_total_suspect' THEN COALESCE(signing_value, current_value) ELSE COALESCE(current_value, signing_value) END AS trusted_native, CASE new_value_flag @@ -1793,13 +2050,17 @@ WITH contract_base AS ( WHEN signing_value IS NOT NULL THEN COALESCE(NULLIF(currency, ''), 'BGN') ELSE COALESCE(NULLIF(current_value_currency, ''), NULLIF(currency, ''), 'BGN') END + WHEN 'annex_total_suspect' THEN CASE + WHEN signing_value IS NOT NULL THEN COALESCE(NULLIF(currency, ''), 'BGN') + ELSE COALESCE(NULLIF(current_value_currency, ''), NULLIF(currency, ''), 'BGN') + END ELSE CASE WHEN current_value IS NOT NULL THEN COALESCE(NULLIF(current_value_currency, ''), NULLIF(currency, ''), 'BGN') ELSE COALESCE(NULLIF(currency, ''), 'BGN') END END AS trusted_currency, CASE - WHEN new_value_flag IN ('value_suspect', 'annex_suspect') OR current_value IS NULL THEN NULL + WHEN new_value_flag IN ('value_suspect', 'annex_suspect', 'annex_total_suspect') OR current_value IS NULL THEN NULL WHEN COALESCE(NULLIF(current_value_currency, ''), NULLIF(currency, ''), 'BGN') = 'EUR' THEN current_value WHEN COALESCE(NULLIF(current_value_currency, ''), NULLIF(currency, ''), 'BGN') = 'BGN' THEN current_value / 1.95583 WHEN fx_rate IS NOT NULL THEN current_value * fx_rate diff --git a/scripts/work-staging-schema.sql b/scripts/work-staging-schema.sql index 9e85b00d..6c403e9c 100644 --- a/scripts/work-staging-schema.sql +++ b/scripts/work-staging-schema.sql @@ -174,6 +174,13 @@ CREATE TABLE raw_amendments ( value_before REAL, -- Стойност преди изменението value_after REAL, -- Стойност след изменението → current_value value_delta REAL, -- Изменение на стойността + -- #305 Tier-2 text-based value correction (computed in TS ingest, packages/ingest/src/amendment-total.ts): + -- value_treatment labels how the основание text reads value_delta ('total_restated' / 'unchanged_restated' + -- / 'genuine_increment', NULL when no signal); value_after_restated is the corrected (true) total when a + -- double-count was confirmed, else NULL. Derive/normalize use COALESCE(value_after_restated, value_after) + -- as the effective after, and skip the arithmetic annex_total_suspect flag when value_treatment IS NOT NULL. + value_treatment TEXT, + value_after_restated REAL, currency TEXT, description TEXT, -- Описание на измененията reason TEXT, -- Причини за изменение (ЗОП основание) diff --git a/tsconfig.base.json b/tsconfig.base.json index 939300bc..bcaee8ac 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -17,6 +17,7 @@ "forceConsistentCasingInFileNames": true, "declaration": false, "sourceMap": true, - "noEmit": true + "noEmit": true, + "allowImportingTsExtensions": true } }