diff --git a/README.md b/README.md index 3a65a504..fb50e4b4 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,7 @@ | Профил на компания | `/companies/[eik]` | Един получател: общо спечелено, от кого, какво | | Договори | `/contracts` | Филтриран списък с договори | | Детайл на договор | `/contracts/[id]` | Една сделка с пълна проследимост до източника | +| Аномалии | `/anomalies` | Договори с автоматични ценови сигнали (над прогнозата, ръст чрез анекси, далеч над типичното) | | Потоци | `/flows` | Парични потоци възложител → компания (суми + брой) | | Търсене | `/search` | По имена, предмет и идентификатори | @@ -47,7 +48,7 @@ Прозрачността върху публичните данни работи днес. Следващите стъпки надграждат същата структура, която вече ги предвижда, затова нищо не се преправя наново: - **Слой „собственици и свързани лица“** — пълно свързване с Търговския регистър. -- **Автоматични проверки** на задания, цени и картели, с **публичен рисков индекс** на всяка поръчка (зелено / жълто / червено) — с публична методология и отворен код. +- **Автоматични проверки** на задания, цени и картели, с **публичен рисков индекс** на всяка поръчка (зелено / жълто / червено) — с публична методология и отворен код. Първата част вече е налице — страницата **Аномалии** (`/anomalies`) с три ценови сигнала и рисков скор на договор. - **AI асистент на български** — разговорен слой над данните (текст и глас): намира и обяснява поръчките и изготвя справки. Освен контрол, целта е и да помага на малкия и средния бизнес да намира лесно подходящите за него поръчки. СИГМА е част от инициативите на МИДТ за прозрачност чрез отворени данни. @@ -66,7 +67,7 @@ pnpm dev # ежедневно: пуска приложението и ETL w `pnpm setup` инсталира зависимостите, прилага D1 миграциите и зарежда малък примерен набор данни (`scripts/seed.sql`) в локалната miniflare база. `pnpm dev` стартира приложението на и ETL worker-а на `:8789`. Пълният корпус се зарежда с `pnpm run import` — изисква свалената EOP емисия в `data/eop` (виж [ETL](#etl)). -> **Статус:** прототип. Приложението и refresh worker-ът работят локално и се деплойват на Cloudflare през GitHub Actions; аналитичният слой (оценка на риска, аномалии, картели) е в плана за развитие. +> **Статус:** прототип. Приложението и refresh worker-ът работят локално и се деплойват на Cloudflare през GitHub Actions; от аналитичния слой са налични първите автоматични ценови проверки (`/anomalies`); останалото (пълен рисков индекс, картели) е в плана за развитие. ## Структура на хранилището diff --git a/apps/web/app/components/SiteHeader.tsx b/apps/web/app/components/SiteHeader.tsx index c091ec57..7cd3fab7 100644 --- a/apps/web/app/components/SiteHeader.tsx +++ b/apps/web/app/components/SiteHeader.tsx @@ -15,6 +15,7 @@ const NAV: NavItem[] = [ { to: '/authorities', label: 'Институции' }, { to: '/companies', label: 'Компании' }, { to: '/contracts', label: 'Договори' }, + { to: '/anomalies', label: 'Аномалии' }, { to: '/analytics', label: 'Анализи', activePaths: [...ANALYTICS_NAV_PATHS] }, { to: '/methodology', label: 'Методология' }, ]; diff --git a/apps/web/app/lib/anomaly-badges.test.ts b/apps/web/app/lib/anomaly-badges.test.ts new file mode 100644 index 00000000..5b5e7883 --- /dev/null +++ b/apps/web/app/lib/anomaly-badges.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, it } from 'vitest'; +import type { AnomalySignals } from '@sigma/api-contract'; +import { anomalyBadges, formatTimes } from './anomaly-badges'; + +const NBSP = '\u00A0'; // non-breaking space, as emitted by @sigma/shared formatters + +const none: AnomalySignals = { + overEstimateRatio: null, + estimatedEur: null, + annexGrowthRatio: null, + priceRatio: null, + peerMedianEur: null, + peerCount: null, + singleBid: false, + noNotice: false, +}; + +describe('formatTimes', () => { + it('renders one decimal with a Bulgarian comma, dropping a trailing ,0', () => { + expect(formatTimes(2.53)).toBe('×2,5'); + expect(formatTimes(12)).toBe('×12'); + expect(formatTimes(1.1)).toBe('×1,1'); + }); + + it('renders extreme ratios as whole numbers with the thousands separator', () => { + expect(formatTimes(104527.7)).toBe(`×104${NBSP}528`); + }); +}); + +describe('anomalyBadges', () => { + it('maps every fired signal in severity order, price signals plain and context soft', () => { + const badges = anomalyBadges({ + overEstimateRatio: 2.5, + estimatedEur: 102258, + annexGrowthRatio: 1.6, + priceRatio: 12, + peerMedianEur: 41666, + peerCount: 120, + singleBid: true, + noNotice: true, + }); + + expect(badges.map((b) => b.key)).toEqual([ + 'over_estimate', + 'annex_growth', + 'price_outlier', + 'single_bid', + 'no_notice', + ]); + expect(badges.map((b) => b.context)).toEqual([false, false, false, true, true]); + + expect(badges[0]).toMatchObject({ + label: '×2,5 над прогнозата', + detail: `(при 102${NBSP}хил.${NBSP}€)`, + }); + expect(badges[1]).toMatchObject({ label: '+60% чрез анекси', detail: null }); + expect(badges[2]!.label).toBe('×12 над типичното'); + expect(badges[2]!.detail).toBe(`(медиана 42${NBSP}хил.${NBSP}€ от 120 договора)`); + expect(badges[3]).toMatchObject({ label: 'единствена оферта', detail: null }); + expect(badges[4]).toMatchObject({ label: 'без обявление', detail: null }); + }); + + it('renders nothing for a signal-less row and omits absent evidence details', () => { + expect(anomalyBadges(none)).toEqual([]); + const noEvidence = anomalyBadges({ ...none, priceRatio: 7.2, peerMedianEur: null }); + expect(noEvidence).toHaveLength(1); + expect(noEvidence[0]).toMatchObject({ label: '×7,2 над типичното', detail: null }); + }); + + it('rounds the annex growth to whole percents', () => { + const badges = anomalyBadges({ ...none, annexGrowthRatio: 1.2345 }); + expect(badges[0]!.label).toBe('+23% чрез анекси'); + }); +}); diff --git a/apps/web/app/lib/anomaly-badges.ts b/apps/web/app/lib/anomaly-badges.ts new file mode 100644 index 00000000..dfc8c753 --- /dev/null +++ b/apps/web/app/lib/anomaly-badges.ts @@ -0,0 +1,69 @@ +// Anomaly signal badges — the pure display mapping from an AnomalyListItem's fired signals to the +// red-flag chips on /anomalies. Kept out of the route component so the copy/formatting is unit +// tested. Formatting is hand-rolled like @sigma/shared/format: workerd does not carry the bg-BG +// Intl data, so no Intl/toLocaleString here. +import type { AnomalySignals } from '@sigma/api-contract'; +import type { AnomalySignalKey } from '@sigma/config'; +import { count, money, signedPct } from '@sigma/shared'; + +export interface AnomalyBadge { + key: AnomalySignalKey; + /** Headline chip text, e.g. „×2,5 над прогнозата". */ + label: string; + /** Baseline evidence rendered de-emphasised inside the chip, e.g. „(при 102 хил. €)". */ + detail: string | null; + /** true → context signal (soft chip variant), never the reason the row exists. */ + context: boolean; +} + +/** „×2,5" / „×12" / „×104 528" — one decimal under 100 (trailing „,0" dropped, comma decimal), + * whole numbers with the thousands NBSP above. */ +export function formatTimes(ratio: number): string { + const body = + ratio >= 100 ? count(Math.round(ratio)) : String(Number(ratio.toFixed(1))).replace('.', ','); + return `×${body}`; +} + +/** + * The chips for one row, in severity order (price signals first, context last). Ratio fields are + * already flag-gated by the query layer (non-null ⇔ the signal fired), so presence alone decides. + */ +export function anomalyBadges(s: AnomalySignals): AnomalyBadge[] { + const badges: AnomalyBadge[] = []; + if (s.overEstimateRatio != null) { + badges.push({ + key: 'over_estimate', + label: `${formatTimes(s.overEstimateRatio)} над прогнозата`, + detail: s.estimatedEur != null ? `(при ${money(s.estimatedEur)})` : null, + context: false, + }); + } + if (s.annexGrowthRatio != null) { + badges.push({ + key: 'annex_growth', + label: `${signedPct(s.annexGrowthRatio - 1, 0)} чрез анекси`, + detail: null, + context: false, + }); + } + if (s.priceRatio != null) { + badges.push({ + key: 'price_outlier', + label: `${formatTimes(s.priceRatio)} над типичното`, + detail: + s.peerMedianEur != null + ? `(медиана ${money(s.peerMedianEur)}${ + s.peerCount != null ? ` от ${count(s.peerCount)} договора` : '' + })` + : null, + context: false, + }); + } + if (s.singleBid) { + badges.push({ key: 'single_bid', label: 'единствена оферта', detail: null, context: true }); + } + if (s.noNotice) { + badges.push({ key: 'no_notice', label: 'без обявление', detail: null, context: true }); + } + return badges; +} diff --git a/apps/web/app/lib/filters.ts b/apps/web/app/lib/filters.ts index 3cf59955..397fa02b 100644 --- a/apps/web/app/lib/filters.ts +++ b/apps/web/app/lib/filters.ts @@ -8,7 +8,7 @@ import { normalizeAuthoritySort, normalizeCompanySort, normalizeContractSort } f import type { CpvCategory } from '@sigma/config'; import type { FilterCategory, FilterGroup, FilterOption } from '../components/FilterRail'; -export const PAGE_SIZE = { contracts: 15, companies: 25, authorities: 25 } as const; +export const PAGE_SIZE = { contracts: 15, companies: 25, authorities: 25, anomalies: 15 } as const; export const MAX_MULTI_VALUES = 50; const KNOWN_SECTORS = new Set(CPV_SECTORS.map((s) => s.code)); @@ -181,6 +181,7 @@ const PARAM_ORDER = [ 'q', 'type', 'kind', + 'signal', 'sector', 'year', 'procedure', diff --git a/apps/web/app/routes.ts b/apps/web/app/routes.ts index 70909b7d..959fafed 100644 --- a/apps/web/app/routes.ts +++ b/apps/web/app/routes.ts @@ -21,6 +21,7 @@ export default [ route('contracts.csv', 'routes/contracts.csv.tsx'), route('contracts/:id.json', 'routes/contract.json.tsx'), route('contracts/:id', 'routes/contract.tsx'), + route('anomalies', 'routes/anomalies.tsx'), route('methodology', 'routes/methodology.tsx'), route('accessibility', 'routes/accessibility.tsx'), route('privacy', 'routes/privacy.tsx'), diff --git a/apps/web/app/routes/anomalies.tsx b/apps/web/app/routes/anomalies.tsx new file mode 100644 index 00000000..fc12b41e --- /dev/null +++ b/apps/web/app/routes/anomalies.tsx @@ -0,0 +1,256 @@ +import { Link, useNavigation, useSearchParams } from 'react-router'; +import { count, date, money } from '@sigma/shared'; +import { getAnomalyFacets, listAnomalies, type AnomalySort } from '@sigma/db'; +import type { AnomalyListItem } from '@sigma/api-contract'; +import type { Route } from './+types/anomalies'; +import { Breadcrumbs } from '../components/Breadcrumbs'; +import { PageHeader } from '../components/PageHeader'; +import { FilterRail, type FilterGroup } from '../components/FilterRail'; +import { ListControls } from '../components/ListControls'; +import { Pagination } from '../components/Pagination'; +import { Callout, Flag } from '../components/ui'; +import { anomalyBadges } from '../lib/anomaly-badges'; +import { + buildSectorGroup, + getMulti, + leaderboardRankOffset, + pageNav, + withParams, + PAGE_SIZE, +} from '../lib/filters'; +import { publicCache } from '../lib/cache'; +import { withDbRetry } from '../lib/retry'; + +const VALUE_BUCKETS = [ + { value: 'lt100k', label: 'Под 100 хил. €' }, + { value: '100k-1m', label: '100 хил. – 1 млн. €' }, + { value: '1m-10m', label: '1 – 10 млн. €' }, + { value: '10m-100m', label: '10 – 100 млн. €' }, + { value: 'gt100m', label: 'Над 100 млн. €' }, +]; + +export function meta(_: Route.MetaArgs) { + return [ + { title: 'Аномалии — СИГМА' }, + { + name: 'description', + content: + 'Автоматични проверки на цените по обществени поръчки: договори над прогнозата, раснали чрез анекси или далеч над типичното за сектора.', + }, + ]; +} + +export function headers() { + return { 'Cache-Control': publicCache(1800) }; +} + +export async function loader({ request, context }: Route.LoaderArgs) { + const sp = new URL(request.url).searchParams; + const params = { + sort: (sp.get('sort') as AnomalySort) || 'score-desc', + signals: getMulti(sp, 'signal'), + years: getMulti(sp, 'year'), + sectors: getMulti(sp, 'sector'), + valueBucket: sp.get('value'), + authority: sp.get('authority'), + bidder: sp.get('bidder'), + cursor: sp.get('cursor'), + pageSize: PAGE_SIZE.anomalies, + }; + const { env } = context.cloudflare; + return withDbRetry(async () => { + const [result, facets] = await Promise.all([ + listAnomalies(env.DB, params), + getAnomalyFacets(env.DB), + ]); + return { result, facets }; + }); +} + +// The fired signals as red-flag chips. The mapping/copy lives in lib/anomaly-badges (unit tested); +// each chip carries its numbers inline, so a row is verifiable at a glance without opening the +// contract. +function SignalFlags({ item }: { item: AnomalyListItem }) { + const badges = anomalyBadges(item.signals); + if (badges.length === 0) return null; + return ( + + {badges.map((b) => ( + + {b.label} + {b.detail && {b.detail}} + + ))} + + ); +} + +export default function Anomalies({ loaderData }: Route.ComponentProps) { + const { result, facets } = loaderData; + const [sp] = useSearchParams(); + const sort = sp.get('sort') ?? 'score-desc'; + const nav = pageNav({ + base: sp, + total: result.total, + pageSize: PAGE_SIZE.anomalies, + nextCursor: result.nextCursor, + prevCursor: result.prevCursor, + }); + const busy = useNavigation().state !== 'idle'; + + const groups: FilterGroup[] = [ + { + key: 'signal', + label: 'Сигнал', + type: 'checkbox', + selected: getMulti(sp, 'signal'), + options: facets.signals.map((s) => ({ value: s.value, label: s.label, count: s.count })), + }, + buildSectorGroup( + facets.sectors.map((s) => ({ value: s.value, label: s.label, count: s.count })), + getMulti(sp, 'sector'), + ), + { + key: 'year', + label: 'Година', + type: 'checkbox', + selected: getMulti(sp, 'year'), + options: facets.years.map((y) => ({ value: y.value, label: y.label, count: y.count })), + }, + { + key: 'value', + label: 'Стойност (в евро)', + type: 'radio', + selected: sp.get('value') ? [sp.get('value')!] : [], + options: VALUE_BUCKETS, + }, + ]; + + const startRank = leaderboardRankOffset(nav.page, PAGE_SIZE.anomalies); + + return ( + <> + +
+ + +
+ +
+ + Намерени {count(result.total)} договора ·{' '} + {money(result.valueEur)} + + } + /> + + {result.items.length === 0 ? ( +

+ Няма резултати за избраните филтри. Изчисти филтрите +

+ ) : ( +
+ + + + + + + + + + + + + {result.items.map((c, i) => ( + + + + + + + + ))} + +
+ Договори с автоматични сигнали за ценови аномалии +
+ # + Договор · СигналиВъзложител · Изпълнител + Дата + + Стойност · Риск +
+ {startRank + i + 1} + + + {c.subject} + + + УНП {c.unp} + {c.isConsortium ? ' · обединение' : ''} + + + + + {c.authorityName}{' '} + възложител + + + {c.bidderDisplayName}{' '} + изпълнител + + + {date(c.signedAt)} + + {money(c.valueEur)} +
+ + {c.score} + /100 + +
+
+ )} + + {result.items.length > 0 && } + + +

+ Как се изчисляват сигналите +

+

+ Три ценови сигнала поставят договор в този списък: подписан ≥ 10% над прогнозата на + възложителя, ръст ≥ 20% чрез анекси или стойност ≥ 5 пъти над медианата на договорите + със същия CPV код. „Единствена оферта“ и „без обявление“ добавят точки към риска, но + сами по себе си не са аномалия. Праговете, изключенията и ограниченията са описани в{' '} + методологията. +

+
+
+
+
+ + ); +} diff --git a/apps/web/app/routes/methodology.tsx b/apps/web/app/routes/methodology.tsx index 85f48842..7f615996 100644 --- a/apps/web/app/routes/methodology.tsx +++ b/apps/web/app/routes/methodology.tsx @@ -36,6 +36,7 @@ const TOC = [ ['money', 'Валута, закръгляване, периоди'], ['identity', 'Имена, ЕИК, УНП'], ['gaps', 'Известни празнини в полетата'], + ['flags', 'Аномалии и сигнали'], ['export', 'Сваляне и достъп до данните'], ['contact', 'Поправки и обратна връзка'], ]; @@ -143,8 +144,10 @@ export default function Methodology({ loaderData }: Route.ComponentProps) {

- СИГМА е изцяло само за четене: не въвежда нови данни, не оценява процедурите и не - маркира фирми като рискови. + СИГМА е изцяло само за четене: не въвежда нови данни и не маркира фирми като + рискови. Автоматичните ценови сигнали (виж раздел „Аномалии и сигнали“) са + детерминистични проверки върху отделни договори, не оценки на фирми или + институции.

@@ -459,15 +462,70 @@ export default function Methodology({ loaderData }: Route.ComponentProps) { -

- Място на изпълнение, собственици и свързани лица и{' '} - рискови сигнали са в процес на разработка за следваща версия — - изискват пълно сливане с допълнителни източници и отделен аналитичен слой. +

+ Място на изпълнение и собственици и свързани лица са в + процес на разработка за следваща версия — изискват пълно сливане с допълнителни + източници. Първите автоматични ценови сигнали вече са налични — виж + следващия раздел.

+
+

9. Аномалии и сигнали

+

+ Страницата Аномалии показва договори, при които + детерминистична проверка върху числата от регистъра е дала сигнал. + Сигналът е индикатор за проверка, не присъда — висока стойност може + да има напълно легитимно обяснение (по-голям обем, опции, индексация). +

+

Три ценови сигнала поставят договор в списъка:

+
    +
  • + Над прогнозата — стойността при подписване е с ≥ 10% над + прогнозната стойност, обявена от самия възложител. Сравняваме само когато + прогнозата покрива точно този договор: прогноза на обособената позиция за + договори по лот, прогноза на поръчката за поръчки без лотове. Рамкови + споразумения и динамични системи (повече договори, отколкото лотове — там + прогнозата е общ таван) се изключват, както и прогнози под 1 хил. € и договори + под 10 хил. €. +
  • +
  • + Ръст чрез анекси — текущата стойност след изменения е с ≥ 20% над + стойността при подписване. ЗОП чл. 116 ограничава повечето изменения до + 10–50%, затова ръст ≥ 50% носи по-висока тежест. +
  • +
  • + Далеч над типичното — стойността е ≥ 5 пъти над медианата на + договорите със същия пълен CPV код (при ≥ 10 такива договора и стойност ≥ 50 + хил. €). Това е най-мекият сигнал: по-голяма поръчка не значи по-лоша цена, + затова показваме медианата и броя съпоставими договори до всяко число. + Медианата се изчислява върху всички съпоставими договори, включително самия + договор — при праг от ≥ 10 съпоставими ефектът е пренебрежим. +
  • +
+

+ Два контекстни сигнала — единствена оферта в състезателна + процедура и възлагане без обявление — добавят точки към риска, но + сами по себе си не поставят договор в списъка. Рискът (0–100) е претеглен сбор: + над прогнозата 25/35/45 т. (≥ 1,1× / 1,5× / 3×), анекси 20/30 т. (≥ 1,2× / 1,5×), + над типичното 15/25 т. (≥ 5× / 10×), единствена оферта 10 т., без обявление 5 т. +

+ +

+ Проверките работят само с числата, публикувани в регистъра — не виждат + количества, единични цени или качество на изпълнението. Договори със съмнителни + стойности в източника („данните се проверяват“) са изключени от проверките + изцяло. Медианите по CPV код се преизчисляват при пълно презареждане на данните, + не в реално време: дневното обновяване преизчислява сигналите само за променените + договори спрямо последните изчислени медиани, а съвсем нова група съпоставими + договори започва да дава сигнал при следващото пълно презареждане. Праговете и + кодът на проверките са публични в хранилището на проекта. +

+
+
+
-

9. Сваляне и достъп до данните

+

10. Сваляне и достъп до данните

Всеки списък може да бъде свален като CSV — точно това, което виждаш, с приложените филтри: @@ -485,7 +543,7 @@ export default function Methodology({ loaderData }: Route.ComponentProps) {

-

10. Поправки и обратна връзка

+

11. Поправки и обратна връзка

Грешките поправяме ръчно при сигнал — двойни записи за институция/компания (изпратете двата ЕИК/линка) или сума, която не отговаря на оригиналния документ diff --git a/apps/web/app/routes/sitemap-pages.tsx b/apps/web/app/routes/sitemap-pages.tsx index ed1f14c7..7629345a 100644 --- a/apps/web/app/routes/sitemap-pages.tsx +++ b/apps/web/app/routes/sitemap-pages.tsx @@ -10,6 +10,7 @@ const PAGES: Page[] = [ { loc: '/companies' }, { loc: '/authorities' }, { loc: '/contracts' }, + { loc: '/anomalies' }, { loc: '/analytics' }, { loc: '/flows' }, { loc: '/network' }, diff --git a/apps/web/app/styles/components.css b/apps/web/app/styles/components.css index 73c7cc7a..23991729 100644 --- a/apps/web/app/styles/components.css +++ b/apps/web/app/styles/components.css @@ -161,6 +161,18 @@ a.flag:hover { color: var(--ink-soft); } +/* Anomaly rows: the fired-signal chip strip under the contract title. The parenthesised numbers + inside a chip (baseline the ratio was measured against) drop the mono-uppercase shout. */ +.signal-flags { + display: block; + margin-top: 4px; +} +.signal-flags .flag-detail { + text-transform: none; + letter-spacing: 0.02em; + font-weight: 400; +} + /* Methodology callout — left rule, paper-warm body, no fill on neutral */ .callout { border-left: 4px solid var(--ink); diff --git a/apps/web/vite.config.ts b/apps/web/vite.config.ts index d74ba7ff..270646c5 100644 --- a/apps/web/vite.config.ts +++ b/apps/web/vite.config.ts @@ -13,6 +13,10 @@ export default defineConfig({ cloudflare({ viteEnvironment: { name: 'ssr' }, persistState: { path: persistPath }, + // Vectorize + AI bindings (added with the assistant feature) cannot be emulated by miniflare + // and would attempt a remote proxy session that requires a Cloudflare login. Disable the + // remote proxy so local dev stays fully offline; the assistant route falls back to a 503. + remoteBindings: false, }), tailwindcss(), reactRouter(), diff --git a/packages/api-contract/src/index.ts b/packages/api-contract/src/index.ts index d8c489e7..fdd7d2f9 100644 --- a/packages/api-contract/src/index.ts +++ b/packages/api-contract/src/index.ts @@ -596,3 +596,57 @@ export interface SearchResults { groups: SearchGroup[]; empty: boolean; } + +// ── Anomalies (automated red-flag screen) ─────────────────────────────────────────────────────── + +/** The fired signals on one anomaly row. Ratio fields are populated ONLY when the corresponding + * signal actually fired (the stored under-threshold ratios stay server-side), so the UI renders a + * badge per non-null field without re-checking thresholds. */ +export interface AnomalySignals { + /** signing value ÷ the authority's own estimate (≥ 1.1 when present). */ + overEstimateRatio: number | null; + /** The comparable estimate in EUR (present with overEstimateRatio). */ + estimatedEur: number | null; + /** current (post-annex) ÷ signing value (≥ 1.2 when present). */ + annexGrowthRatio: number | null; + /** contract value ÷ CPV-code peer median (≥ 5 when present). */ + priceRatio: number | null; + peerMedianEur: number | null; + peerCount: number | null; + /** Context: one offer in a competitive procedure. */ + singleBid: boolean; + /** Context: direct / no-notice procedure. */ + noNotice: boolean; +} + +export interface AnomalyListItem { + id: string; // /contracts/:id + subject: string; + unp: string; + sectorCode: string | null; + authoritySlug: string; + authorityName: string; + bidderSlug: string; + bidderName: string; + bidderDisplayName: string; + bidderKind: EntityKind; + isConsortium: boolean; + signedAt: string | null; + valueEur: number; + /** 0–100 weighted signal score (see /methodology). */ + score: number; + signals: AnomalySignals; +} + +/** Headline for the anomalies list under the active filter. */ +export interface AnomaliesSummary { + total: number; + valueEur: number; +} + +/** Global (unfiltered) per-signal row counts for the filter rail. */ +export interface AnomalyFacets { + signals: FacetCount[]; + sectors: FacetCount[]; + years: FacetCount[]; +} diff --git a/packages/config/src/index.ts b/packages/config/src/index.ts index 483dd008..da9d76c3 100644 --- a/packages/config/src/index.ts +++ b/packages/config/src/index.ts @@ -407,3 +407,74 @@ export function regionByName(name: string | null | undefined): BgRegion | null { if (!name) return null; return BG_REGION_BY_NAME.get(name.trim()) ?? null; } + +// ── Anomaly signals (contract_anomalies flag_* → display metadata) ────────────────────────────── +// +// The automated red-flag checks precomputed into `contract_anomalies` (scripts/precompute.sql §7). +// Deterministic rules over source figures — thresholds live in the SQL; this is the display +// taxonomy (filter facet labels, badge captions, methodology copy). `qualifying`: true = the signal +// alone puts a contract on the anomaly screen; false = context-only (adds score, never creates a +// row). Signals are indicators for public scrutiny, never verdicts of wrongdoing. + +export type AnomalySignalKey = + | 'over_estimate' + | 'annex_growth' + | 'price_outlier' + | 'single_bid' + | 'no_notice'; + +export interface AnomalySignal { + key: AnomalySignalKey; + /** Short Bulgarian label for the filter facet / badge. */ + label: string; + /** One-line Bulgarian explanation (title/tooltip + methodology row). */ + description: string; + /** true = fires a row on the anomaly screen; false = context that only adds to the score. */ + qualifying: boolean; +} + +export const ANOMALY_SIGNALS: readonly AnomalySignal[] = [ + { + key: 'over_estimate', + label: 'Над прогнозата', + description: + 'Договорът е подписан на стойност поне 10% над прогнозната стойност, обявена от самия възложител.', + qualifying: true, + }, + { + key: 'annex_growth', + label: 'Ръст чрез анекси', + description: + 'Стойността е нараснала с поне 20% след подписването чрез изменения (анекси) по договора.', + qualifying: true, + }, + { + key: 'price_outlier', + label: 'Далеч над типичното', + description: + 'Стойността е поне 5 пъти над медианата на съпоставимите договори със същия CPV код (при поне 10 такива).', + qualifying: true, + }, + { + key: 'single_bid', + label: 'Единствена оферта', + description: 'Само една оферта в състезателна процедура — липса на реална конкуренция.', + qualifying: false, + }, + { + key: 'no_notice', + label: 'Без обявление', + description: 'Възложено чрез пряко договаряне или процедура без предварително обявление.', + qualifying: false, + }, +]; + +const ANOMALY_SIGNAL_BY_KEY = new Map( + ANOMALY_SIGNALS.map((s) => [s.key, s]), +); + +/** Resolve an anomaly-signal key to its display metadata, or null for an unknown key. */ +export function anomalySignal(key: string | null | undefined): AnomalySignal | null { + if (!key) return null; + return ANOMALY_SIGNAL_BY_KEY.get(key) ?? null; +} diff --git a/packages/db/migrations/0005_anomalies.sql b/packages/db/migrations/0005_anomalies.sql new file mode 100644 index 00000000..3eb1aff3 --- /dev/null +++ b/packages/db/migrations/0005_anomalies.sql @@ -0,0 +1,55 @@ +-- Sigma — anomaly screen storage: cpv_price_stats + contract_anomalies. +-- +-- Numbered incremental migration (the repo's current convention for post-0000 schema changes; +-- 0002/0004 are claimed by in-flight PRs, so this takes 0005). Both tables are populated by +-- scripts/precompute.sql §7 on every full import and scoped-refreshed by the daily slice +-- (scripts/refresh-slice.sql, @refresh-batch anomalies); methodology lives on /methodology#flags. +-- +-- IF NOT EXISTS on purpose: unlike 0000_init (fresh-database assumption), these tables may already +-- exist on a running database — precompute.sql and the refresh batch create them defensively so the +-- ETL keeps working on databases migrated before this file landed. The guards make this migration a +-- safe no-op there and keep both creation paths convergent. + +-- Per full CPV code: peer count + median contract value over clean rows. Feeds the price-outlier +-- signal in contract_anomalies; rebuilt on full import (precompute.sql), kept as-is by the daily +-- slice (medians drift negligibly within a day; a brand-new cohort appears on the next full import). +CREATE TABLE IF NOT EXISTS cpv_price_stats ( + cpv_code TEXT PRIMARY KEY, + peers INTEGER NOT NULL, -- clean contracts sharing this full CPV code + median_eur REAL NOT NULL -- median amount_eur of those contracts +); + +-- Anomaly screen: one row per contract with at least one fired PRICE signal (over-estimate / annex +-- growth / price outlier). Signals are INDICATORS for public scrutiny, not verdicts — thresholds and +-- exclusions are documented in scripts/precompute.sql §7 and on /methodology. The `flag_*` columns +-- are the authoritative triggers (ratios are stored whenever computable, even under threshold); +-- single_bid / no_notice are context-only and never create a row by themselves. Denormalised +-- amount/date/sector/party columns keep list filtering on this small table (no 190k-row scans). +CREATE TABLE IF NOT EXISTS contract_anomalies ( + contract_id TEXT PRIMARY KEY REFERENCES contracts(id), + score INTEGER NOT NULL, -- 0–100 weighted sum of fired signals (see precompute §7) + rank_value REAL NOT NULL, -- score-major, amount-minor sort key (score×1e12 + amount) + flag_over_estimate INTEGER NOT NULL DEFAULT 0, -- signed ≥ +10% above the authority's own estimate + flag_annex_growth INTEGER NOT NULL DEFAULT 0, -- grew ≥ +20% via annexes + flag_price_outlier INTEGER NOT NULL DEFAULT 0, -- ≥5× the CPV-code median (peers ≥ 10, ≥ €50k) + flag_single_bid INTEGER NOT NULL DEFAULT 0, -- one offer in a competitive procedure (context) + flag_no_notice INTEGER NOT NULL DEFAULT 0, -- direct / no-notice procedure (context) + over_estimate_ratio REAL, -- signing_value_eur / estimated_eur (when comparable) + estimated_eur REAL, -- the comparable estimate (lot-level, or single-lot tender) + annex_growth_ratio REAL, -- current_value_eur / signing_value_eur (annexed rows) + price_ratio REAL, -- amount_eur / peer_median_eur (peers ≥ 10) + peer_median_eur REAL, + peer_count INTEGER, + amount_eur REAL NOT NULL, -- copied from contracts for filter/sort locality + signed_at TEXT, + cpv_division TEXT, -- 2-digit CPV division (sector facet) + authority_id TEXT NOT NULL, + bidder_id TEXT NOT NULL +); + +-- Anomaly screen: default sort (score-major/amount-minor) + the value/date sorts and party scoping. +CREATE INDEX IF NOT EXISTS idx_anomalies_rank ON contract_anomalies(rank_value DESC); +CREATE INDEX IF NOT EXISTS idx_anomalies_amount ON contract_anomalies(amount_eur); +CREATE INDEX IF NOT EXISTS idx_anomalies_signed ON contract_anomalies(signed_at); +CREATE INDEX IF NOT EXISTS idx_anomalies_authority ON contract_anomalies(authority_id); +CREATE INDEX IF NOT EXISTS idx_anomalies_bidder ON contract_anomalies(bidder_id); diff --git a/packages/db/src/anomaly-parity.test.ts b/packages/db/src/anomaly-parity.test.ts new file mode 100644 index 00000000..2efa78a7 --- /dev/null +++ b/packages/db/src/anomaly-parity.test.ts @@ -0,0 +1,149 @@ +/// +// Guard against silent drift of the anomaly derive/scoring SQL, which is intentionally duplicated +// between scripts/precompute.sql §7 (full rebuild) and scripts/refresh-slice.sql (@refresh-batch +// anomalies, scoped daily re-derive). The two copies must stay byte-identical between the +// @anomaly-derive markers — only the FROM/WHERE scoping between the marked regions may differ +// (full corpus vs touched slice). A threshold or weight changed in one file but not the other +// fails HERE instead of silently disagreeing in production. +import { readFileSync } from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; +import { PROCEDURE_GROUPS } from '@sigma/config'; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); +const precomputeSql = readFileSync(resolve(root, 'scripts/precompute.sql'), 'utf8').replaceAll('\r\n', '\n'); +const refreshSliceSql = readFileSync(resolve(root, 'scripts/refresh-slice.sql'), 'utf8').replaceAll('\r\n', '\n'); + +/** The text strictly between a `-- @ begin…` line and its `-- @ end…` line. + * Extraction is EXCLUSIVE: the marker lines themselves are not part of the returned string. + * The begin regex consumes the entire begin line + newline; the end regex anchors to the start + * of the end line so `end.index` points before the end marker. */ +function markedRegion(sql: string, name: string, file: string): string { + const begin = new RegExp(`^-- @${name} begin[^\\n]*\\n`, 'm').exec(sql); + const end = new RegExp(`^-- @${name} end[^\\n]*$`, 'm').exec(sql); + if (!begin || !end) throw new Error(`missing @${name} begin/end markers in ${file}`); + return sql.slice(begin.index + begin[0].length, end.index); +} + +describe('anomaly derive parity (precompute.sql §7 ↔ refresh-slice.sql)', () => { + it('keeps the shared derive/scoring block byte-identical', () => { + const a = markedRegion(precomputeSql, 'anomaly-derive', 'scripts/precompute.sql'); + const b = markedRegion(refreshSliceSql, 'anomaly-derive', 'scripts/refresh-slice.sql'); + expect(a).toBe(b); + // The block really is the one carrying every threshold/weight, not an empty region. + for (const invariant of [ + 'INSERT INTO contract_anomalies (', + '>= 1.10 THEN 1 ELSE 0 END AS flag_over', + '>= 1.20 THEN 1 ELSE 0 END AS flag_annex', + 'x.ratio >= 5 AND x.amount_eur >= 50000', + 'ps.peers >= 10', + "'Пряко договаряне'", + ]) { + expect(a).toContain(invariant); + } + }); + + it('keeps the shared qualifying-rows tail byte-identical', () => { + const a = markedRegion(precomputeSql, 'anomaly-derive-tail', 'scripts/precompute.sql'); + const b = markedRegion(refreshSliceSql, 'anomaly-derive-tail', 'scripts/refresh-slice.sql'); + expect(a).toBe(b); + expect(a).toContain('WHERE flag_over = 1 OR flag_annex = 1 OR flag_outlier = 1'); + }); + + it('computes score once and derives rank_value from it (no duplicated scoring expression)', () => { + const region = markedRegion(precomputeSql, 'anomaly-derive', 'scripts/precompute.sql'); + // score is computed once via MIN(100, …) + expect(region).toContain(') AS score,'); + // rank_value is derived from score, not by repeating the scoring expression + expect(region).toContain('score * 1e12 + amount_eur AS rank_value,'); + // exactly ONE occurrence of MIN(100, — a second copy would reintroduce the duplication + expect((region.match(/MIN\(100,/g) ?? []).length).toBe(1); + }); + + it('marker extraction is exclusive — marker lines are never part of the extracted region', () => { + // The begin/end lines intentionally differ between files (they name the other file). + // If extraction were inclusive, the parity comparison would fail even with identical logic. + const region = markedRegion(precomputeSql, 'anomaly-derive', 'scripts/precompute.sql'); + expect(region).not.toContain('@anomaly-derive'); + // Specifically, the file-name references on the marker lines must not leak in. + expect(region).not.toContain('refresh-slice.sql'); + expect(region).not.toContain('precompute.sql'); + }); + + it('covers all competitive procedure_type values from PROCEDURE_GROUPS in the single_bid clause', () => { + // PROCEDURE_GROUPS is the canonical source of truth for procedure_type strings. + // Any value with competitive===true must appear in the single_bid IN list so a + // one-bid competitive procedure is caught regardless of wording variant. + const region = markedRegion(precomputeSql, 'anomaly-derive', 'scripts/precompute.sql'); + const competitiveTypes = PROCEDURE_GROUPS.filter((g) => g.competitive === true).flatMap( + (g) => g.types, + ); + for (const type of competitiveTypes) { + expect(region).toContain(`'${type}'`); + } + }); + + it('covers all direct/no-notice procedure_type values from PROCEDURE_GROUPS in the no_notice clause', () => { + const region = markedRegion(precomputeSql, 'anomaly-derive', 'scripts/precompute.sql'); + const directTypes = PROCEDURE_GROUPS.filter((g) => g.competitive === false).flatMap( + (g) => g.types, + ); + for (const type of directTypes) { + expect(region).toContain(`'${type}'`); + } + }); + + it('refresh_touched_contracts is created in the setup batch, before the anomalies batch depends on it', () => { + // The @refresh-batch setup batch must unconditionally create refresh_touched_contracts before + // @refresh-batch anomalies — DELETE … WHERE contract_id IN (SELECT id FROM refresh_touched_contracts) + // and the scoped derive filter both require the table to exist. + const setupStart = refreshSliceSql.indexOf('-- @refresh-batch setup'); + const anomaliesStart = refreshSliceSql.indexOf('-- @refresh-batch anomalies'); + const createTable = refreshSliceSql.indexOf('CREATE TABLE refresh_touched_contracts'); + expect(setupStart).toBeGreaterThanOrEqual(0); + expect(anomaliesStart).toBeGreaterThanOrEqual(0); + // refresh_touched_contracts must be defined before the anomalies batch needs it + expect(createTable).toBeGreaterThan(setupStart); + expect(createTable).toBeLessThan(anomaliesStart); + // The DELETE in the anomalies batch that depends on it must exist + expect(refreshSliceSql).toContain( + 'DELETE FROM contract_anomalies WHERE contract_id IN (SELECT id FROM refresh_touched_contracts)', + ); + }); + + it('keeps the clean-corpus scope predicate in both scoping sections', () => { + // The scoping sections between the markers legitimately differ (full corpus vs touched slice), + // but both must select only clean rows. + const scope = "WHERE c.value_flag = 'ok' AND c.amount_eur > 0"; + expect(precomputeSql).toContain(scope); + expect(refreshSliceSql).toContain(scope); + }); +}); + +describe('anomaly derive parity (precompute.sql §7 ↔ refresh-slice.sql)', () => { + it('keeps the shared derive/scoring block byte-identical', () => { + const a = markedRegion(precomputeSql, 'anomaly-derive', 'scripts/precompute.sql'); + const b = markedRegion(refreshSliceSql, 'anomaly-derive', 'scripts/refresh-slice.sql'); + expect(a).toBe(b); + // The block really is the one carrying every threshold/weight, not an empty region. + for (const invariant of [ + 'INSERT INTO contract_anomalies (', + '>= 1.10 THEN 1 ELSE 0 END AS flag_over', + '>= 1.20 THEN 1 ELSE 0 END AS flag_annex', + 'x.ratio >= 5 AND x.amount_eur >= 50000', + 'ps.peers >= 10', + "'Пряко договаряне'", + ]) { + expect(a).toContain(invariant); + } + }); + + it('keeps the clean-corpus scope predicate in both scoping sections', () => { + // The scoping sections between the markers legitimately differ (full corpus vs touched slice), + // but both must select only clean rows. + const scope = "WHERE c.value_flag = 'ok' AND c.amount_eur > 0"; + expect(precomputeSql).toContain(scope); + expect(refreshSliceSql).toContain(scope); + }); +}); diff --git a/packages/db/src/anomaly-precompute.test.ts b/packages/db/src/anomaly-precompute.test.ts new file mode 100644 index 00000000..fbb3839b --- /dev/null +++ b/packages/db/src/anomaly-precompute.test.ts @@ -0,0 +1,182 @@ +/// +// Behavioural tests for the anomaly derive/scoring SQL (scripts/precompute.sql §7) against a real +// SQLite database: a tiny seeded corpus exercises each signal's threshold, the framework exclusion +// and the qualifying-row rule, then §7 runs verbatim from the repo file. Complements +// anomaly-parity.test.ts (which pins the refresh-slice copy to this exact block). Requires the +// sqlite3 CLI (devcontainer/CI), like the other *.sql-driven suites. +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { beforeAll, describe, expect, it } from 'vitest'; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); +const migrationsDir = resolve(root, 'packages/db/migrations'); +const precomputePath = resolve(root, 'scripts/precompute.sql'); + +function sqlite(dbPath: string, sql: string): string { + return execFileSync('sqlite3', ['-bail', dbPath], { input: sql, encoding: 'utf8' }); +} + +function sqliteJson(dbPath: string, sql: string): T[] { + const out = execFileSync('sqlite3', ['-json', dbPath, sql], { encoding: 'utf8' }).trim(); + return out ? (JSON.parse(out) as T[]) : []; +} + +function readScript(dbPath: string, path: string): void { + execFileSync('sqlite3', ['-bail', dbPath], { input: `.read ${path}\n`, stdio: 'pipe' }); +} + +/** §7 of precompute.sql — the anomaly build — exactly as shipped. */ +function anomalySection(): string { + const sql = readFileSync(precomputePath, 'utf8'); + const start = sql.indexOf('-- ── 7) Anomaly screen'); + const end = sql.indexOf('-- Summary (last result set'); + if (start === -1 || end === -1 || end <= start) + throw new Error('precompute.sql §7 markers not found'); + return sql.slice(start, end); +} + +// One authority/bidder; per-case tenders+contracts. Peer cohort 45233120: eleven 10 000 € contracts +// + one 120 000 € outlier (the cohort median intentionally includes the outlier itself — n=12, the +// middle pair stays 10 000). The other cases use unique CPV codes so peers < 10 keeps ratio NULL. +const BASE_SEED = ` +INSERT INTO authorities (id, name) VALUES ('auth:100', 'Тест Възложител'); +INSERT INTO bidders (id, name) VALUES ('eik:200', 'Тест ООД'); +`; + +const SEED = ` +-- over-estimate ≥3× + single bid in a competitive procedure → 45 + 10 = 55 +INSERT INTO tenders (id, source_id, title, authority_id, cpv_code, estimated_value, currency, procedure_type) +VALUES ('t:13', 'UNP-13', 'Служба по чистота', 'auth:100', '90911200', 100000, 'BGN', 'Открита процедура'); +INSERT INTO contracts (id, tender_id, bidder_id, amount, amount_eur, signing_value_eur, bids_received, signed_at, value_flag) +VALUES ('c:13', 't:13', 'eik:200', 312933, 160000, 160000, 1, '2024-05-01', 'ok'); + +-- annex growth ≥1.5× + no-notice procedure → 30 + 5 = 35 +INSERT INTO tenders (id, source_id, title, authority_id, cpv_code, procedure_type) +VALUES ('t:14', 'UNP-14', 'Правни услуги', 'auth:100', '79111000', 'Пряко договаряне'); +INSERT INTO contracts (id, tender_id, bidder_id, amount, amount_eur, signing_value_eur, current_value_eur, annex_count, signed_at, value_flag) +VALUES ('c:14', 't:14', 'eik:200', 195583, 100000, 100000, 160000, 1, '2024-06-01', 'ok'); + +-- framework/DPS: two awards on a single-lot tender → the estimate is a ceiling, MUST NOT flag +INSERT INTO tenders (id, source_id, title, authority_id, cpv_code, estimated_value, currency, procedure_type) +VALUES ('t:15', 'UNP-15', 'Хотелско настаняване', 'auth:100', '55100000', 1000, 'BGN', 'Открита процедура'); +INSERT INTO contracts (id, tender_id, bidder_id, amount, amount_eur, signing_value_eur, signed_at, value_flag) +VALUES ('c:15a', 't:15', 'eik:200', 97792, 50000, 50000, '2024-07-01', 'ok'), + ('c:15b', 't:15', 'eik:200', 97792, 50000, 50000, '2024-07-01', 'ok'); + +-- context-only single bid, no price signal → MUST NOT create a row +INSERT INTO tenders (id, source_id, title, authority_id, cpv_code, procedure_type) +VALUES ('t:16', 'UNP-16', 'Градски превоз', 'auth:100', '60100000', 'Публично състезание'); +INSERT INTO contracts (id, tender_id, bidder_id, amount, amount_eur, bids_received, signed_at, value_flag) +VALUES ('c:16', 't:16', 'eik:200', 39117, 20000, 1, '2024-08-01', 'ok'); +`; + +function peerCohortSeed(): string { + const rows: string[] = []; + for (let i = 1; i <= 11; i += 1) { + rows.push(` +INSERT INTO tenders (id, source_id, title, authority_id, cpv_code, procedure_type) +VALUES ('t:${i}', 'UNP-${i}', 'Пътна маркировка ${i}', 'auth:100', '45233120', 'Открита процедура'); +INSERT INTO contracts (id, tender_id, bidder_id, amount, amount_eur, bids_received, signed_at, value_flag) +VALUES ('c:${i}', 't:${i}', 'eik:200', 19558, 10000, 3, '2024-01-0${(i % 9) + 1}', 'ok');`); + } + rows.push(` +INSERT INTO tenders (id, source_id, title, authority_id, cpv_code, procedure_type) +VALUES ('t:12', 'UNP-12', 'Пътна маркировка 12', 'auth:100', '45233120', 'Открита процедура'); +INSERT INTO contracts (id, tender_id, bidder_id, amount, amount_eur, bids_received, signed_at, value_flag) +VALUES ('c:12', 't:12', 'eik:200', 234700, 120000, 3, '2024-02-01', 'ok');`); + return rows.join('\n'); +} + +interface AnomalyRow { + contract_id: string; + score: number; + flag_over_estimate: number; + flag_annex_growth: number; + flag_price_outlier: number; + flag_single_bid: number; + flag_no_notice: number; + over_estimate_ratio: number | null; + estimated_eur: number | null; + annex_growth_ratio: number | null; + price_ratio: number | null; + peer_median_eur: number | null; + peer_count: number | null; + cpv_division: string | null; +} + +describe('anomaly precompute (§7) behaviour', () => { + let dir: string; + let dbPath: string; + const rowsById = new Map(); + + beforeAll(() => { + dir = mkdtempSync(resolve(tmpdir(), 'sigma-anomaly-')); + dbPath = resolve(dir, 'test.sqlite'); + for (const file of readdirSync(migrationsDir).filter((f) => f.endsWith('.sql')).sort()) + readScript(dbPath, resolve(migrationsDir, file)); + sqlite(dbPath, BASE_SEED); + sqlite(dbPath, peerCohortSeed()); + sqlite(dbPath, SEED); + const sectionPath = resolve(dir, 'section7.sql'); + writeFileSync(sectionPath, anomalySection(), 'utf8'); + readScript(dbPath, sectionPath); + for (const row of sqliteJson(dbPath, 'SELECT * FROM contract_anomalies')) + rowsById.set(row.contract_id, row); + return () => rmSync(dir, { recursive: true, force: true }); + }); + + it('computes the cohort median over the full CPV code, including the contract itself', () => { + const stats = sqliteJson<{ peers: number; median_eur: number }>( + dbPath, + "SELECT peers, median_eur FROM cpv_price_stats WHERE cpv_code = '45233120'", + ); + expect(stats).toEqual([{ peers: 12, median_eur: 10000 }]); + }); + + it('flags a ≥10× price outlier at 25 points with the peer evidence stored', () => { + const row = rowsById.get('c:12')!; + expect(row).toMatchObject({ + score: 25, + flag_price_outlier: 1, + flag_over_estimate: 0, + flag_annex_growth: 0, + flag_single_bid: 0, + flag_no_notice: 0, + price_ratio: 12, + peer_median_eur: 10000, + peer_count: 12, + cpv_division: '45', + }); + }); + + it('does not flag the 10 000 € cohort peers (ratio 1, under every floor)', () => { + for (let i = 1; i <= 11; i += 1) expect(rowsById.has(`c:${i}`)).toBe(false); + }); + + it('flags ≥3× over the own estimate at 45 points, plus 10 for the single bid', () => { + const row = rowsById.get('c:13')!; + expect(row).toMatchObject({ score: 55, flag_over_estimate: 1, flag_single_bid: 1 }); + // est 100 000 BGN → 51 129.55 € at the peg; 160 000 / 51 129.55 ≈ 3.13 (≥ 3 → the 45 tier). + expect(row.estimated_eur!).toBeCloseTo(100000 / 1.95583, 2); + expect(row.over_estimate_ratio!).toBeGreaterThanOrEqual(3); + expect(row.price_ratio).toBeNull(); // peers < 10 → no cohort comparison + }); + + it('flags ≥1.5× annex growth at 30 points, plus 5 for the no-notice procedure', () => { + const row = rowsById.get('c:14')!; + expect(row).toMatchObject({ score: 35, flag_annex_growth: 1, flag_no_notice: 1 }); + expect(row.annex_growth_ratio!).toBeCloseTo(1.6, 6); + }); + + it('excludes framework/DPS call-offs from the estimate comparison (awards > lots)', () => { + expect(rowsById.has('c:15a')).toBe(false); + expect(rowsById.has('c:15b')).toBe(false); + }); + + it('never creates a row from context signals alone', () => { + expect(rowsById.has('c:16')).toBe(false); + }); +}); diff --git a/packages/db/src/migrations.test.ts b/packages/db/src/migrations.test.ts index 3e26faba..61e845b7 100644 --- a/packages/db/src/migrations.test.ts +++ b/packages/db/src/migrations.test.ts @@ -1,14 +1,13 @@ /// import { execFileSync } from 'node:child_process'; -import { mkdtempSync, rmSync } from 'node:fs'; +import { mkdtempSync, readdirSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { dirname, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { describe, expect, it } from 'vitest'; const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); -const migration0 = resolve(root, 'packages/db/migrations/0000_init.sql'); -const migration1 = resolve(root, 'packages/db/migrations/0001_flow_pairs_bidder_index.sql'); +const migrationsDir = resolve(root, 'packages/db/migrations'); function sqlite(dbPath: string, sql: string): string { return execFileSync('sqlite3', [dbPath], { input: sql, encoding: 'utf8' }); @@ -19,14 +18,20 @@ function readScript(dbPath: string, path: string): void { } describe('served migrations', () => { - // 0000_init remains the complete base served schema. Later migrations must be additive over that - // base so initial setup (`wrangler d1 migrations apply`) and ETL ships keep the same table shape. - it('builds the served schema from the migration chain', () => { + // The consolidated 0000_init carries the pre-production schema; later schema changes land as + // numbered incremental migrations (the same order `wrangler d1 migrations apply` uses). This + // guards that the chain applied in filename order yields the complete served schema — amendments + // history, the OCDS parties projection, the EOP tenderId column, the anomaly-screen tables — and + // carries no raw_* staging (that lives only in work-staging-schema.sql, applied to the work DB). + it('builds the complete served schema from the migration chain', () => { const dir = mkdtempSync(resolve(tmpdir(), 'sigma-migrations-')); const dbPath = resolve(dir, 'test.sqlite'); try { - readScript(dbPath, migration0); - readScript(dbPath, migration1); + const migrationFiles = readdirSync(migrationsDir) + .filter((f) => f.endsWith('.sql')) + .sort(); + expect(migrationFiles[0]).toBe('0000_init.sql'); + for (const file of migrationFiles) readScript(dbPath, resolve(migrationsDir, file)); expect( sqlite( @@ -68,6 +73,14 @@ describe('served migrations', () => { ).trim(), ).toBe('1'); + // The anomaly screen reads precomputed tables — shipped by the 0005_anomalies migration. + expect( + sqlite(dbPath, "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name IN ('contract_anomalies', 'cpv_price_stats');").trim(), + ).toBe('2'); + expect( + sqlite(dbPath, "SELECT COUNT(*) FROM pragma_table_info('contract_anomalies') WHERE name='rank_value' AND \"notnull\"=1;").trim(), + ).toBe('1'); + // The served schema must never carry raw_* staging tables. expect( sqlite(dbPath, "SELECT COUNT(*) FROM sqlite_master WHERE name LIKE 'raw_%';").trim(), diff --git a/packages/db/src/queries/anomalies.test.ts b/packages/db/src/queries/anomalies.test.ts new file mode 100644 index 00000000..2d9d4c37 --- /dev/null +++ b/packages/db/src/queries/anomalies.test.ts @@ -0,0 +1,184 @@ +import { describe, expect, it } from 'vitest'; +import { getAnomalyFacets, listAnomalies, anomaliesSummary } from './anomalies'; + +const anomalyRow = { + id: 'c:1', + subject: 'Доставка на материали', + unp: 'UNP-1', + cpv_division: '45', + signed_at: '2024-03-01', + amount_eur: 500000, + score: 60, + flag_over_estimate: 1, + over_estimate_ratio: 2.5, + estimated_eur: 200000, + flag_annex_growth: 0, + annex_growth_ratio: 1.05, // stored under threshold — must NOT surface (flag is 0) + flag_price_outlier: 1, + price_ratio: 12, + peer_median_eur: 41666, + peer_count: 120, + flag_single_bid: 1, + flag_no_notice: 0, + authority_id: 'auth:000695089', + authority_name: 'Министерство на финансите', + bidder_id: 'eik:111111111', + bidder_name: 'ТЕСТ ООД', + bidder_kind: 'company' as const, + sort_value: 60e12 + 500000, +}; + +function fakeDb(rows: (typeof anomalyRow)[] = [anomalyRow]): D1Database { + return { + prepare(sql: string) { + return { + bind() { + return this; + }, + async all() { + return { results: (sql.includes('1=0') ? [] : rows) as T[] }; + }, + async first() { + const total = sql.includes('1=0') ? 0 : rows.length; + return { total, eur: total ? 500000 : 0 } as T; + }, + }; + }, + } as D1Database; +} + +// SQL-capturing fake: records every prepared statement so the filter-shape tests assert on the +// query text (there is no real D1 here), mirroring flows.test.ts. +function spyDb(): { db: D1Database; sql: string[] } { + const sql: string[] = []; + const db = { + prepare(q: string) { + sql.push(q); + return { + bind() { + return this; + }, + async all() { + return { results: [] as T[] }; + }, + async first() { + return { total: 0, eur: 0 } as T; + }, + }; + }, + } as D1Database; + return { db, sql }; +} + +describe('listAnomalies', () => { + it('maps a row and surfaces ratios only for fired flags', async () => { + const page = await listAnomalies(fakeDb(), { pageSize: 10 }); + + expect(page.total).toBe(1); + const item = page.items[0]!; + expect(item.id).toBe('1'); // contractSlug strips the 'c:' domain prefix + expect(item.score).toBe(60); + expect(item.valueEur).toBe(500000); + // over_estimate + price_outlier fired → ratios present; annex flag is 0 → its stored + // under-threshold ratio stays hidden. + expect(item.signals.overEstimateRatio).toBe(2.5); + expect(item.signals.estimatedEur).toBe(200000); + expect(item.signals.priceRatio).toBe(12); + expect(item.signals.peerMedianEur).toBe(41666); + expect(item.signals.peerCount).toBe(120); + expect(item.signals.annexGrowthRatio).toBeNull(); + expect(item.signals.singleBid).toBe(true); + expect(item.signals.noNotice).toBe(false); + }); + + it('returns no rows for an undecodable bidder slug', async () => { + const page = await listAnomalies(fakeDb(), { bidder: 'n%', pageSize: 10 }); + + expect(page.items).toEqual([]); + expect(page.total).toBe(0); + }); + + it('falls back to the default sort instead of throwing (sort=toString)', async () => { + await expect( + listAnomalies(fakeDb(), { sort: 'toString' as never, pageSize: 10 }), + ).resolves.toBeDefined(); + }); + + it('filters signals via the precomputed flag columns (no re-stated thresholds)', async () => { + const { db, sql } = spyDb(); + await listAnomalies(db, { signals: ['over_estimate', 'single_bid'], pageSize: 10 }); + + const listSql = sql.find((q) => q.includes('FROM contract_anomalies an'))!; + expect(listSql).toContain('an.flag_over_estimate = 1 OR an.flag_single_bid = 1'); + }); + + it('yields an empty result (not an unfiltered list) when only unknown signal keys are given', async () => { + const { db, sql } = spyDb(); + await listAnomalies(db, { signals: ['constructor'], pageSize: 10 }); + + const listSql = sql.find((q) => q.includes('FROM contract_anomalies an'))!; + expect(listSql).toContain('1=0'); + }); + + it('keeps every filter predicate on the anomaly table (an.*)', async () => { + const { db, sql } = spyDb(); + await listAnomalies(db, { + years: ['2024'], + sectors: ['45'], + valueBucket: '1m-10m', + authority: '000695089', + pageSize: 10, + }); + + const listSql = sql.find((q) => q.includes('FROM contract_anomalies an'))!; + expect(listSql).toContain("substr(an.signed_at, 1, 4) IN (?)"); + expect(listSql).toContain('an.cpv_division IN (?)'); + expect(listSql).toContain('an.amount_eur >= ? AND an.amount_eur < ?'); + expect(listSql).toContain('an.authority_id = ?'); + }); +}); + +describe('anomaliesSummary', () => { + it('ignores a reserved value-bucket key instead of a destructure TypeError (value=toString)', async () => { + await expect( + anomaliesSummary(fakeDb(), { valueBucket: 'toString' }), + ).resolves.toMatchObject({ total: 1 }); + }); +}); + +describe('getAnomalyFacets', () => { + it('maps signal counts in display order and drops empty buckets', async () => { + const db = { + prepare(sql: string) { + return { + async all() { + if (sql.includes('cpv_division')) { + return { results: [{ key: '45', contracts: 7 }] as T[] }; + } + return { results: [{ key: '2024', contracts: 9 }] as T[] }; + }, + async first() { + return { + over_estimate: 5, + annex_growth: 0, + price_outlier: 3, + single_bid: 2, + no_notice: 0, + } as T; + }, + }; + }, + } as D1Database; + + const facets = await getAnomalyFacets(db); + + expect(facets.signals.map((s) => s.value)).toEqual([ + 'over_estimate', + 'price_outlier', + 'single_bid', + ]); + expect(facets.signals[0]).toMatchObject({ count: 5 }); + expect(facets.sectors[0]).toMatchObject({ value: '45', count: 7 }); + expect(facets.years[0]).toMatchObject({ value: '2024', label: '2024', count: 9 }); + }); +}); diff --git a/packages/db/src/queries/anomalies.ts b/packages/db/src/queries/anomalies.ts new file mode 100644 index 00000000..38293a12 --- /dev/null +++ b/packages/db/src/queries/anomalies.ts @@ -0,0 +1,326 @@ +// Anomalies — the automated red-flag screen. The list reads the precomputed `contract_anomalies` +// table (built by scripts/precompute.sql §7, scoped-refreshed daily) and joins the domain tables +// only for the 15 displayed rows; every filter/sort/summary predicate stays on the small anomaly +// table. Signals are indicators for public scrutiny, never verdicts — see /methodology. + +import type { + AnomaliesSummary, + AnomalyFacets, + AnomalyListItem, + AnomalySignals, + FacetCount, + Page, +} from '@sigma/api-contract'; +import { ANOMALY_SIGNALS, CPV_SECTORS } from '@sigma/config'; +import { cleanName, entityName } from '@sigma/shared'; +import { authoritySlug, bidderIdFromSlug, companySlug, contractSlug } from './identity'; +import { filterSignature, keyset, pageCursors } from './keyset'; +import { lookup } from './lookup'; + +export type AnomalySort = 'score-desc' | 'value-desc' | 'value-asc' | 'date-desc' | 'date-asc'; + +export interface AnomalyListParams { + sort?: AnomalySort; + signals?: string[]; + years?: string[]; + sectors?: string[]; + valueBucket?: string | null; + authority?: string | null; // authority ЕИК (slug) + bidder?: string | null; // bidder slug + cursor?: string | null; + pageSize?: number; +} + +export const ANOMALY_FILTER_KEYS = [ + 'signals', + 'years', + 'sectors', + 'valueBucket', + 'authority', + 'bidder', +] as const satisfies readonly (keyof AnomalyListParams)[]; + +// rank_value = score×1e12 + amount_eur (precomputed) — score-major, value-minor, so equal scores +// surface the big money first without a composite keyset cursor. +const SORTS: Record = lookup({ + 'score-desc': { expr: 'an.rank_value', dir: 'desc' }, + 'value-desc': { expr: 'COALESCE(an.amount_eur, -1)', dir: 'desc' }, + 'value-asc': { expr: 'COALESCE(an.amount_eur, 1e18)', dir: 'asc' }, + 'date-desc': { expr: "COALESCE(an.signed_at, '')", dir: 'desc' }, + 'date-asc': { expr: "COALESCE(an.signed_at, '9999-99')", dir: 'asc' }, +}); + +// Signal filter key → the authoritative trigger column (thresholds are baked into the flags at +// precompute time, so the WHERE never re-states them). +const SIGNAL_COLUMNS: Record = lookup({ + over_estimate: 'an.flag_over_estimate', + annex_growth: 'an.flag_annex_growth', + price_outlier: 'an.flag_price_outlier', + single_bid: 'an.flag_single_bid', + no_notice: 'an.flag_no_notice', +}); + +const VALUE_BUCKETS: Record = lookup({ + lt100k: [0, 100_000], + '100k-1m': [100_000, 1_000_000], + '1m-10m': [1_000_000, 10_000_000], + '10m-100m': [10_000_000, 100_000_000], + gt100m: [100_000_000, null], +}); + +const qs = (n: number) => Array.from({ length: n }, () => '?').join(', '); + +interface AnomalyRow { + id: string; + subject: string; + unp: string; + cpv_division: string | null; + signed_at: string | null; + amount_eur: number; + score: number; + flag_over_estimate: number; + over_estimate_ratio: number | null; + estimated_eur: number | null; + flag_annex_growth: number; + annex_growth_ratio: number | null; + flag_price_outlier: number; + price_ratio: number | null; + peer_median_eur: number | null; + peer_count: number | null; + flag_single_bid: number; + flag_no_notice: number; + authority_id: string; + authority_name: string; + bidder_id: string; + bidder_name: string; + bidder_kind: 'company' | 'consortium'; +} + +const SELECT = ` + SELECT an.contract_id AS id, COALESCE(NULLIF(c.contract_subject, ''), t.title) AS subject, + t.source_id AS unp, an.cpv_division, an.signed_at, an.amount_eur, an.score, + an.flag_over_estimate, an.over_estimate_ratio, an.estimated_eur, + an.flag_annex_growth, an.annex_growth_ratio, + an.flag_price_outlier, an.price_ratio, an.peer_median_eur, an.peer_count, + an.flag_single_bid, an.flag_no_notice, + an.authority_id, a.name AS authority_name, + an.bidder_id, b.name AS bidder_name, b.kind AS bidder_kind`; +const FROM = ` + FROM contract_anomalies an + JOIN contracts c ON c.id = an.contract_id + JOIN tenders t ON t.id = c.tender_id + JOIN authorities a ON a.id = an.authority_id + JOIN bidders b ON b.id = an.bidder_id`; +// Summary/facet aggregates never need the display joins. +const FROM_BARE = ` FROM contract_anomalies an`; + +/** + * Build the WHERE fragment (with a leading ' WHERE ') + params shared by list and summary. Every + * predicate targets `an.*` so aggregates ride the small table. Keep consumed filter keys in sync + * with ANOMALY_FILTER_KEYS and anomalyFilterSignature(). + */ +function buildFilters(p: AnomalyListParams): { sql: string; params: unknown[] } { + const where: string[] = []; + const params: unknown[] = []; + if (p.signals?.length) { + const cols = p.signals.map((s) => SIGNAL_COLUMNS[s]).filter((c): c is string => Boolean(c)); + if (cols.length) where.push(`(${cols.map((c) => `${c} = 1`).join(' OR ')})`); + else where.push('1=0'); // only unknown signal keys → empty result, not an unfiltered list + } + if (p.years?.length) { + where.push(`substr(an.signed_at, 1, 4) IN (${qs(p.years.length)})`); + params.push(...p.years); + } + if (p.sectors?.length) { + where.push(`an.cpv_division IN (${qs(p.sectors.length)})`); + params.push(...p.sectors); + } + const bucket = p.valueBucket ? VALUE_BUCKETS[p.valueBucket] : undefined; + if (bucket) { + const [lo, hi] = bucket; + where.push(hi == null ? `an.amount_eur >= ?` : `(an.amount_eur >= ? AND an.amount_eur < ?)`); + params.push(lo); + if (hi != null) params.push(hi); + } + if (p.authority) { + where.push(`an.authority_id = ?`); + params.push('auth:' + p.authority); + } + if (p.bidder) { + const id = bidderIdFromSlug(p.bidder); + if (id) { + where.push(`an.bidder_id = ?`); + params.push(id); + } else { + where.push('1=0'); + } + } + return { sql: where.length ? ' WHERE ' + where.join(' AND ') : '', params }; +} + +function anomalyFilterSignature(p: AnomalyListParams): string { + const bidder = p.bidder ? (bidderIdFromSlug(p.bidder) ?? `invalid:${p.bidder}`) : null; + const filters = { + signals: p.signals, + years: p.years, + sectors: p.sectors, + valueBucket: p.valueBucket, + authority: p.authority, + bidder, + } satisfies Record<(typeof ANOMALY_FILTER_KEYS)[number], unknown>; + return filterSignature(filters); +} + +function toItem(r: AnomalyRow): AnomalyListItem { + const authorityName = cleanName(r.authority_name); + const bidderName = cleanName(r.bidder_name); + // Ratios surface only when the corresponding flag fired, so the UI can render one badge per + // non-null field without re-stating thresholds. + const signals: AnomalySignals = { + overEstimateRatio: r.flag_over_estimate === 1 ? r.over_estimate_ratio : null, + estimatedEur: r.flag_over_estimate === 1 ? r.estimated_eur : null, + annexGrowthRatio: r.flag_annex_growth === 1 ? r.annex_growth_ratio : null, + priceRatio: r.flag_price_outlier === 1 ? r.price_ratio : null, + peerMedianEur: r.flag_price_outlier === 1 ? r.peer_median_eur : null, + peerCount: r.flag_price_outlier === 1 ? r.peer_count : null, + singleBid: r.flag_single_bid === 1, + noNotice: r.flag_no_notice === 1, + }; + return { + id: contractSlug(r.id), + subject: r.subject, + unp: r.unp, + sectorCode: r.cpv_division, + authoritySlug: authoritySlug(r.authority_id), + authorityName, + bidderSlug: companySlug(r.bidder_id), + bidderName, + bidderDisplayName: entityName(bidderName, r.bidder_kind), + bidderKind: r.bidder_kind, + isConsortium: r.bidder_kind === 'consortium', + signedAt: r.signed_at, + valueEur: r.amount_eur, + score: r.score, + signals, + }; +} + +export interface AnomalyListResult extends Page { + valueEur: number; +} + +export async function listAnomalies( + db: D1Database, + p: AnomalyListParams, +): Promise { + const sort = SORTS[p.sort as keyof typeof SORTS] ?? SORTS['score-desc']; + const pageSize = p.pageSize ?? 15; + const filters = buildFilters(p); + const signature = anomalyFilterSignature(p); + const ks = keyset({ + sortCol: sort.expr, + idCol: 'an.contract_id', + dir: sort.dir, + cursor: p.cursor, + filterSignature: signature, + allowedSortCols: Object.values(SORTS).map((s) => s.expr), + }); + + const conds = [filters.sql ? filters.sql.slice(7) : '', ks.whereSql] + .filter(Boolean) + .join(' AND '); + const sql = `${SELECT}, ${sort.expr} AS sort_value ${FROM}${conds ? ' WHERE ' + conds : ''} ${ks.orderSql} LIMIT ?`; + // The page and its headline are independent — one round-trip instead of two sequential ones. + const [{ results }, summary] = await Promise.all([ + db + .prepare(sql) + .bind(...filters.params, ...ks.params, pageSize + 1) + .all(), + anomaliesSummary(db, p), + ]); + + const hasMore = results.length > pageSize; + let rows = results.slice(0, pageSize); + if (ks.reverse) rows = rows.reverse(); + + const cursors = pageCursors({ + rows: rows.map((r) => ({ sortValue: r.sort_value, id: r.id })), + hasMore, + incomingCursor: p.cursor, + cursor: ks.cursor, + sortToken: ks.cursorToken, + }); + + return { + items: rows.map(toItem), + total: summary.total, + valueEur: summary.valueEur, + nextCursor: cursors.nextCursor, + prevCursor: cursors.prevCursor, + }; +} + +/** Row count + flagged value for the current filter (the list headline). */ +export async function anomaliesSummary( + db: D1Database, + p: AnomalyListParams, +): Promise { + const filters = buildFilters(p); + const row = await db + .prepare( + `SELECT COUNT(*) AS total, COALESCE(SUM(an.amount_eur), 0) AS eur${FROM_BARE}${filters.sql}`, + ) + .bind(...filters.params) + .first<{ total: number; eur: number }>(); + return { total: row?.total ?? 0, valueEur: row?.eur ?? 0 }; +} + +/** + * Rail facets — global (unfiltered) counts over contract_anomalies, mirroring the contracts rail. + * Signals keep the ANOMALY_SIGNALS display order; sectors/years render only non-empty buckets + * (rows without a parseable year simply match no year filter). + */ +export async function getAnomalyFacets(db: D1Database): Promise { + const [signalRow, sectorRows, yearRows] = await Promise.all([ + db + .prepare( + `SELECT SUM(flag_over_estimate) AS over_estimate, SUM(flag_annex_growth) AS annex_growth, + SUM(flag_price_outlier) AS price_outlier, SUM(flag_single_bid) AS single_bid, + SUM(flag_no_notice) AS no_notice${FROM_BARE}`, + ) + .first>(), + db + .prepare( + `SELECT cpv_division AS key, COUNT(*) AS contracts${FROM_BARE} + WHERE cpv_division IS NOT NULL GROUP BY cpv_division`, + ) + .all<{ key: string; contracts: number }>(), + db + .prepare( + `SELECT substr(an.signed_at, 1, 4) AS key, COUNT(*) AS contracts${FROM_BARE} + WHERE substr(an.signed_at, 1, 4) GLOB '[0-9][0-9][0-9][0-9]' GROUP BY key`, + ) + .all<{ key: string; contracts: number }>(), + ]); + + const signals: FacetCount[] = ANOMALY_SIGNALS.map((s) => ({ + value: s.key, + label: s.label, + count: Number(signalRow?.[s.key] ?? 0), + })).filter((f) => f.count > 0); + + const sectorByCode = new Map(sectorRows.results.map((r) => [r.key, r.contracts])); + const sectors: FacetCount[] = CPV_SECTORS.map((s) => ({ + value: s.code, + label: s.short ?? s.label, + count: sectorByCode.get(s.code) ?? 0, + })) + .filter((f) => f.count > 0) + .sort((a, b) => b.count - a.count); + + const years: FacetCount[] = yearRows.results + .sort((a, b) => b.key.localeCompare(a.key)) + .map((r) => ({ value: r.key, label: r.key, count: r.contracts })); + + return { signals, sectors, years }; +} diff --git a/packages/db/src/queries/index.ts b/packages/db/src/queries/index.ts index 2ed922e7..cb7e13f4 100644 --- a/packages/db/src/queries/index.ts +++ b/packages/db/src/queries/index.ts @@ -11,6 +11,7 @@ export * from './methodology'; export * from './companies'; export * from './authorities'; export * from './contracts'; +export * from './anomalies'; export * from './flows'; export * from './network'; export * from './trend'; diff --git a/scripts/precompute.sql b/scripts/precompute.sql index d52642d1..77edba1e 100644 --- a/scripts/precompute.sql +++ b/scripts/precompute.sql @@ -167,6 +167,136 @@ FROM contracts c JOIN tenders t ON t.id = c.tender_id JOIN authorities a ON a.id JOIN bidders b ON b.id = c.bidder_id WHERE COALESCE(NULLIF(c.contract_subject, ''), t.title) IS NOT NULL; +-- ── 7) Anomaly screen: cpv_price_stats + contract_anomalies ───────────────────────────────────── +-- Automated red-flag indicators over the clean corpus (value_flag = 'ok', amount_eur > 0). A row in +-- contract_anomalies means at least one PRICE signal fired; signals are indicators for public +-- scrutiny, never verdicts. Methodology (mirrored on /methodology): +-- • over_estimate — signing value ≥ +10% above the authority's OWN pre-tender estimate. Compared +-- only when the estimate covers exactly this award: lot-level estimate for lot-scoped awards, +-- tender estimate for single-lot tenders; framework/DPS call-offs (more awards than lots — the +-- estimate is the whole ceiling, cf. details.ts frameworkAwards) are excluded, as are estimates +-- under €1k and signed values under €10k (noise floors). BGN/EUR only (foreign-currency +-- estimates have no stored rate). +-- • annex_growth — current (post-annex) value ≥ +20% over the signing value, annexed rows only. +-- ЗОП чл. 116 caps most modifications at +10/50%, so ≥1.5 is scored higher. +-- • price_outlier — contract value ≥ 5× the median of ≥10 clean contracts sharing the same FULL +-- CPV code, and ≥ €50k. A scope-vs-price proxy (a bigger buy is not a worse price), hence the +-- softest weight; the median + peer count are stored so every ratio is inspectable. The peer set +-- includes the contract itself (negligible at ≥ 10 peers). Medians are rebuilt HERE, on full +-- import only — the daily slice re-derives touched contracts against the last computed medians. +-- • single_bid / no_notice — competition context (one offer in a competitive procedure / a +-- direct no-notice procedure). Context only: they add score but never create a row. +-- Score = over_estimate 25/35/45 (≥1.1/1.5/3×) + annex_growth 20/30 (≥1.2/1.5×) + +-- price_outlier 15/25 (≥5/10×) + single_bid 10 + no_notice 5, capped at 100. +-- The derive/scoring block between the @anomaly-derive markers is shared verbatim with +-- scripts/refresh-slice.sql (@refresh-batch anomalies) and guarded byte-identical by +-- packages/db/src/anomaly-parity.test.ts — edit both files together. + +CREATE TABLE IF NOT EXISTS cpv_price_stats ( + cpv_code TEXT PRIMARY KEY, peers INTEGER NOT NULL, median_eur REAL NOT NULL +); +DELETE FROM cpv_price_stats; +INSERT INTO cpv_price_stats (cpv_code, peers, median_eur) +SELECT cpv, MAX(n), AVG(eur) FROM ( + SELECT t.cpv_code AS cpv, c.amount_eur AS eur, + ROW_NUMBER() OVER (PARTITION BY t.cpv_code ORDER BY c.amount_eur) AS rn, + COUNT(*) OVER (PARTITION BY t.cpv_code) AS n + FROM contracts c JOIN tenders t ON t.id = c.tender_id + WHERE c.value_flag = 'ok' AND c.amount_eur > 0 AND COALESCE(t.cpv_code, '') <> '' +) +WHERE rn IN ((n + 1) / 2, (n + 2) / 2) +GROUP BY cpv; + +CREATE TABLE IF NOT EXISTS contract_anomalies ( + contract_id TEXT PRIMARY KEY REFERENCES contracts(id), + score INTEGER NOT NULL, rank_value REAL NOT NULL, + flag_over_estimate INTEGER NOT NULL DEFAULT 0, flag_annex_growth INTEGER NOT NULL DEFAULT 0, + flag_price_outlier INTEGER NOT NULL DEFAULT 0, flag_single_bid INTEGER NOT NULL DEFAULT 0, + flag_no_notice INTEGER NOT NULL DEFAULT 0, + over_estimate_ratio REAL, estimated_eur REAL, annex_growth_ratio REAL, + price_ratio REAL, peer_median_eur REAL, peer_count INTEGER, + amount_eur REAL NOT NULL, signed_at TEXT, cpv_division TEXT, + authority_id TEXT NOT NULL, bidder_id TEXT NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_anomalies_rank ON contract_anomalies(rank_value DESC); +CREATE INDEX IF NOT EXISTS idx_anomalies_amount ON contract_anomalies(amount_eur); +CREATE INDEX IF NOT EXISTS idx_anomalies_signed ON contract_anomalies(signed_at); +CREATE INDEX IF NOT EXISTS idx_anomalies_authority ON contract_anomalies(authority_id); +CREATE INDEX IF NOT EXISTS idx_anomalies_bidder ON contract_anomalies(bidder_id); +DELETE FROM contract_anomalies; +-- @anomaly-derive begin (byte-identical with refresh-slice.sql; see anomaly-parity.test.ts) +INSERT INTO contract_anomalies ( + contract_id, score, rank_value, + flag_over_estimate, flag_annex_growth, flag_price_outlier, flag_single_bid, flag_no_notice, + over_estimate_ratio, estimated_eur, annex_growth_ratio, price_ratio, peer_median_eur, peer_count, + amount_eur, signed_at, cpv_division, authority_id, bidder_id) +SELECT + id, score, score * 1e12 + amount_eur AS rank_value, + flag_over, flag_annex, flag_outlier, single_bid, no_notice, + over_ratio, est_eur, growth, ratio, median_eur, peers, + amount_eur, signed_at, cpv_division, authority_id, bidder_id +FROM ( + SELECT + id, + MIN(100, + CASE WHEN flag_over = 1 THEN CASE WHEN over_ratio >= 3 THEN 45 WHEN over_ratio >= 1.5 THEN 35 ELSE 25 END ELSE 0 END + + CASE WHEN flag_annex = 1 THEN CASE WHEN growth >= 1.5 THEN 30 ELSE 20 END ELSE 0 END + + CASE WHEN flag_outlier = 1 THEN CASE WHEN ratio >= 10 THEN 25 ELSE 15 END ELSE 0 END + + CASE WHEN single_bid = 1 THEN 10 ELSE 0 END + + CASE WHEN no_notice = 1 THEN 5 ELSE 0 END) AS score, + flag_over, flag_annex, flag_outlier, single_bid, no_notice, + over_ratio, est_eur, growth, ratio, median_eur, peers, + amount_eur, signed_at, cpv_division, authority_id, bidder_id + FROM ( + SELECT x.*, + CASE WHEN x.est_eur >= 1000 AND x.paid_eur >= 10000 AND x.paid_eur / x.est_eur >= 1.10 THEN 1 ELSE 0 END AS flag_over, + CASE WHEN x.est_eur > 0 THEN x.paid_eur / x.est_eur END AS over_ratio, + CASE WHEN x.growth >= 1.20 THEN 1 ELSE 0 END AS flag_annex, + CASE WHEN x.ratio >= 5 AND x.amount_eur >= 50000 THEN 1 ELSE 0 END AS flag_outlier + FROM ( + SELECT c.id, c.amount_eur, c.signed_at, + substr(t.cpv_code, 1, 2) AS cpv_division, t.authority_id, c.bidder_id, + COALESCE(c.signing_value_eur, c.amount_eur) AS paid_eur, + -- The comparable estimate: only when it covers exactly this award (see header note). + CASE WHEN aw.n <= MAX(COALESCE(t.num_lots, 0), 1) THEN + CASE + WHEN c.lot_id IS NOT NULL AND l.estimated_value > 0 + AND COALESCE(l.value_currency, t.currency, 'BGN') IN ('BGN', 'EUR') + THEN CASE WHEN COALESCE(l.value_currency, t.currency, 'BGN') = 'EUR' + THEN l.estimated_value ELSE l.estimated_value / 1.95583 END + WHEN c.lot_id IS NULL AND COALESCE(t.num_lots, 1) <= 1 AND t.estimated_value > 0 + AND COALESCE(t.currency, 'BGN') IN ('BGN', 'EUR') + THEN CASE WHEN COALESCE(t.currency, 'BGN') = 'EUR' + THEN t.estimated_value ELSE t.estimated_value / 1.95583 END + END + END AS est_eur, + CASE WHEN c.annex_count > 0 AND c.signing_value_eur > 0 AND c.current_value_eur > 0 + THEN c.current_value_eur / c.signing_value_eur END AS growth, + CASE WHEN ps.peers >= 10 AND ps.median_eur > 0 THEN c.amount_eur / ps.median_eur END AS ratio, + ps.median_eur, ps.peers, + CASE WHEN c.bids_received = 1 AND t.procedure_type IN ( + 'Открита процедура', 'Ограничена процедура', 'Ограничена процедура по ДСП', + 'Ограничена процедура по КС', 'Публично състезание', 'Състезателна процедура с договаряне', + 'Събиране на оферти с обява') THEN 1 ELSE 0 END AS single_bid, + CASE WHEN t.procedure_type IN ( + 'Договаряне без предварително обявление', 'Пряко договаряне', + 'Договаряне без предварителна покана за участие', + 'Договаряне без публикуване на обявление за поръчка') THEN 1 ELSE 0 END AS no_notice + FROM contracts c + JOIN tenders t ON t.id = c.tender_id + LEFT JOIN lots l ON l.id = c.lot_id + LEFT JOIN cpv_price_stats ps ON ps.cpv_code = t.cpv_code +-- @anomaly-derive end (the FROM/WHERE scoping below legitimately differs: full corpus here, touched slice there) + LEFT JOIN (SELECT tender_id, COUNT(*) AS n FROM contracts GROUP BY tender_id) aw + ON aw.tender_id = c.tender_id + WHERE c.value_flag = 'ok' AND c.amount_eur > 0 +-- @anomaly-derive-tail begin (byte-identical with refresh-slice.sql) + ) x + WHERE flag_over = 1 OR flag_annex = 1 OR flag_outlier = 1 + ) +); +-- @anomaly-derive-tail end + -- Summary (last result set printed by `wrangler d1 execute`) SELECT (SELECT contracts FROM home_totals) AS home_contracts, @@ -177,4 +307,6 @@ SELECT (SELECT COUNT(*) FROM sector_totals) AS sector_rows, (SELECT COUNT(*) FROM flow_pairs) AS flow_rows, (SELECT COUNT(*) FROM search_index) AS search_rows, - (SELECT COUNT(*) FROM contracts WHERE signing_value_eur IS NOT NULL) AS signing_eur_rows; + (SELECT COUNT(*) FROM contracts WHERE signing_value_eur IS NOT NULL) AS signing_eur_rows, + (SELECT COUNT(*) FROM cpv_price_stats) AS cpv_stat_rows, + (SELECT COUNT(*) FROM contract_anomalies) AS anomaly_rows; diff --git a/scripts/refresh-slice.sql b/scripts/refresh-slice.sql index e051a7ed..647b2ce9 100644 --- a/scripts/refresh-slice.sql +++ b/scripts/refresh-slice.sql @@ -1299,6 +1299,106 @@ FROM contracts c JOIN tenders t ON t.id = c.tender_id JOIN authorities a ON a.id WHERE c.amount_eur IS NOT NULL GROUP BY t.authority_id, c.bidder_id; +-- @refresh-batch anomalies +-- Scoped re-derive of the anomaly screen for touched contracts. The derive/scoring block between +-- the @anomaly-derive markers is shared verbatim with scripts/precompute.sql §7 and guarded +-- byte-identical by packages/db/src/anomaly-parity.test.ts — edit both files together. +-- cpv_price_stats is intentionally NOT rebuilt here: medians are refreshed only on full import, so +-- between imports touched contracts are re-derived against the LAST computed medians (untouched +-- contracts keep their flags, and a cohort newly crossing peers ≥ 10 starts flagging on the next +-- full import). Documented on /methodology#flags. The CREATE guards let the slice run against a +-- database created before the anomaly tables existed. +CREATE TABLE IF NOT EXISTS cpv_price_stats ( + cpv_code TEXT PRIMARY KEY, peers INTEGER NOT NULL, median_eur REAL NOT NULL +); +CREATE TABLE IF NOT EXISTS contract_anomalies ( + contract_id TEXT PRIMARY KEY REFERENCES contracts(id), + score INTEGER NOT NULL, rank_value REAL NOT NULL, + flag_over_estimate INTEGER NOT NULL DEFAULT 0, flag_annex_growth INTEGER NOT NULL DEFAULT 0, + flag_price_outlier INTEGER NOT NULL DEFAULT 0, flag_single_bid INTEGER NOT NULL DEFAULT 0, + flag_no_notice INTEGER NOT NULL DEFAULT 0, + over_estimate_ratio REAL, estimated_eur REAL, annex_growth_ratio REAL, + price_ratio REAL, peer_median_eur REAL, peer_count INTEGER, + amount_eur REAL NOT NULL, signed_at TEXT, cpv_division TEXT, + authority_id TEXT NOT NULL, bidder_id TEXT NOT NULL +); +DELETE FROM contract_anomalies WHERE contract_id IN (SELECT id FROM refresh_touched_contracts); +-- @anomaly-derive begin (byte-identical with precompute.sql §7; see anomaly-parity.test.ts) +INSERT INTO contract_anomalies ( + contract_id, score, rank_value, + flag_over_estimate, flag_annex_growth, flag_price_outlier, flag_single_bid, flag_no_notice, + over_estimate_ratio, estimated_eur, annex_growth_ratio, price_ratio, peer_median_eur, peer_count, + amount_eur, signed_at, cpv_division, authority_id, bidder_id) +SELECT + id, score, score * 1e12 + amount_eur AS rank_value, + flag_over, flag_annex, flag_outlier, single_bid, no_notice, + over_ratio, est_eur, growth, ratio, median_eur, peers, + amount_eur, signed_at, cpv_division, authority_id, bidder_id +FROM ( + SELECT + id, + MIN(100, + CASE WHEN flag_over = 1 THEN CASE WHEN over_ratio >= 3 THEN 45 WHEN over_ratio >= 1.5 THEN 35 ELSE 25 END ELSE 0 END + + CASE WHEN flag_annex = 1 THEN CASE WHEN growth >= 1.5 THEN 30 ELSE 20 END ELSE 0 END + + CASE WHEN flag_outlier = 1 THEN CASE WHEN ratio >= 10 THEN 25 ELSE 15 END ELSE 0 END + + CASE WHEN single_bid = 1 THEN 10 ELSE 0 END + + CASE WHEN no_notice = 1 THEN 5 ELSE 0 END) AS score, + flag_over, flag_annex, flag_outlier, single_bid, no_notice, + over_ratio, est_eur, growth, ratio, median_eur, peers, + amount_eur, signed_at, cpv_division, authority_id, bidder_id + FROM ( + SELECT x.*, + CASE WHEN x.est_eur >= 1000 AND x.paid_eur >= 10000 AND x.paid_eur / x.est_eur >= 1.10 THEN 1 ELSE 0 END AS flag_over, + CASE WHEN x.est_eur > 0 THEN x.paid_eur / x.est_eur END AS over_ratio, + CASE WHEN x.growth >= 1.20 THEN 1 ELSE 0 END AS flag_annex, + CASE WHEN x.ratio >= 5 AND x.amount_eur >= 50000 THEN 1 ELSE 0 END AS flag_outlier + FROM ( + SELECT c.id, c.amount_eur, c.signed_at, + substr(t.cpv_code, 1, 2) AS cpv_division, t.authority_id, c.bidder_id, + COALESCE(c.signing_value_eur, c.amount_eur) AS paid_eur, + -- The comparable estimate: only when it covers exactly this award (see header note). + CASE WHEN aw.n <= MAX(COALESCE(t.num_lots, 0), 1) THEN + CASE + WHEN c.lot_id IS NOT NULL AND l.estimated_value > 0 + AND COALESCE(l.value_currency, t.currency, 'BGN') IN ('BGN', 'EUR') + THEN CASE WHEN COALESCE(l.value_currency, t.currency, 'BGN') = 'EUR' + THEN l.estimated_value ELSE l.estimated_value / 1.95583 END + WHEN c.lot_id IS NULL AND COALESCE(t.num_lots, 1) <= 1 AND t.estimated_value > 0 + AND COALESCE(t.currency, 'BGN') IN ('BGN', 'EUR') + THEN CASE WHEN COALESCE(t.currency, 'BGN') = 'EUR' + THEN t.estimated_value ELSE t.estimated_value / 1.95583 END + END + END AS est_eur, + CASE WHEN c.annex_count > 0 AND c.signing_value_eur > 0 AND c.current_value_eur > 0 + THEN c.current_value_eur / c.signing_value_eur END AS growth, + CASE WHEN ps.peers >= 10 AND ps.median_eur > 0 THEN c.amount_eur / ps.median_eur END AS ratio, + ps.median_eur, ps.peers, + CASE WHEN c.bids_received = 1 AND t.procedure_type IN ( + 'Открита процедура', 'Ограничена процедура', 'Ограничена процедура по ДСП', + 'Ограничена процедура по КС', 'Публично състезание', 'Състезателна процедура с договаряне', + 'Събиране на оферти с обява') THEN 1 ELSE 0 END AS single_bid, + CASE WHEN t.procedure_type IN ( + 'Договаряне без предварително обявление', 'Пряко договаряне', + 'Договаряне без предварителна покана за участие', + 'Договаряне без публикуване на обявление за поръчка') THEN 1 ELSE 0 END AS no_notice + FROM contracts c + JOIN tenders t ON t.id = c.tender_id + LEFT JOIN lots l ON l.id = c.lot_id + LEFT JOIN cpv_price_stats ps ON ps.cpv_code = t.cpv_code +-- @anomaly-derive end (the FROM/WHERE scoping below legitimately differs: full corpus there, touched slice here) + LEFT JOIN (SELECT tender_id, COUNT(*) AS n FROM contracts + WHERE tender_id IN (SELECT tender_id FROM contracts WHERE id IN (SELECT id FROM refresh_touched_contracts)) + GROUP BY tender_id) aw + ON aw.tender_id = c.tender_id + WHERE c.value_flag = 'ok' AND c.amount_eur > 0 + AND c.id IN (SELECT id FROM refresh_touched_contracts) +-- @anomaly-derive-tail begin (byte-identical with precompute.sql §7) + ) x + WHERE flag_over = 1 OR flag_annex = 1 OR flag_outlier = 1 + ) +); +-- @anomaly-derive-tail end + -- @refresh-batch entity-search-index DELETE FROM search_index WHERE kind = 'company'; INSERT INTO search_index (kind, ref, title, ident, subtitle, amount)