diff --git a/README.md b/README.md index 3a65a504..52cba0d9 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ | Потоци | `/flows` | Парични потоци възложител → компания (суми + брой) | | Търсене | `/search` | По имена, предмет и идентификатори | -Списъците имат CSV експорт (`/contracts.csv`, `/companies.csv`, `/authorities.csv`), всеки договор — JSON изглед (`/contracts/[id].json`). Има и страници за методология, достъпност, поверителност и impressum, плюс `robots.txt` и sitemap файлове. +Списъците имат CSV експорт (`/contracts.csv`, `/companies.csv`, `/authorities.csv`), всеки договор — JSON изглед (`/contracts/[id].json`), а профилите на институции и компании — RSS фийд с най-новите им договори (`.rss`). Има и страници за методология, достъпност, поверителност и impressum, плюс `robots.txt` и sitemap файлове. ## Накъде върви diff --git a/apps/web/app/lib/feed.test.ts b/apps/web/app/lib/feed.test.ts new file mode 100644 index 00000000..8c2454de --- /dev/null +++ b/apps/web/app/lib/feed.test.ts @@ -0,0 +1,157 @@ +import { describe, expect, it } from 'vitest'; +import type { ContractListItem } from '@sigma/api-contract'; +import { contractRssItem, rssDate, rssFeed, xmlEscape } from './feed'; + +function item(overrides: Partial = {}): ContractListItem { + return { + id: 'e:UNP-1:2:eik:111111111', + subject: 'Доставка на техника', + unp: 'UNP-1', + sectorCode: '30', + euFunded: false, + isConsortium: false, + authoritySlug: '123456789', + authorityName: 'Община Пример', + bidderSlug: '111111111', + bidderName: 'Фирма ЕООД', + bidderDisplayName: 'Фирма ЕООД', + bidderKind: 'company', + procedureLabel: 'Открита процедура', + signedAt: '2026-05-15', + publishedAt: null, + bidsReceived: 3, + valueEur: 12345.67, + ...overrides, + }; +} + +describe('xmlEscape', () => { + it('escapes the five XML special characters', () => { + expect(xmlEscape(``)).toBe('<a & "b" & 'c'>'); + }); + it('passes ordinary Bulgarian text through unchanged', () => { + expect(xmlEscape('Община „Пример“ - договори')).toBe('Община „Пример“ - договори'); + }); + it('drops XML-1.0-invalid control characters but keeps tab / newline / CR', () => { + const c = (n: number) => String.fromCharCode(n); + // U+0000, U+0008, U+001F are illegal in XML 1.0 → stripped; the surrounding text survives. + expect(xmlEscape(`a${c(0)}b${c(8)}c${c(0x1f)}d`)).toBe('abcd'); + // TAB (U+0009), LF (U+000A), CR (U+000D) are legal → preserved verbatim. + expect(xmlEscape(`x${c(9)}y${c(10)}z${c(13)}w`)).toBe(`x${c(9)}y${c(10)}z${c(13)}w`); + }); +}); + +describe('rssDate', () => { + it('renders an ISO day as RFC 822', () => { + expect(rssDate('2026-05-15')).toBe('Fri, 15 May 2026 00:00:00 GMT'); + }); + it('returns null for null, malformed, and impossible dates', () => { + expect(rssDate(null)).toBeNull(); + expect(rssDate('')).toBeNull(); + expect(rssDate('15.05.2026')).toBeNull(); + expect(rssDate('2026-13-45')).toBeNull(); + }); +}); + +describe('contractRssItem', () => { + it('builds an authority-feed item around the winning bidder', () => { + const rss = contractRssItem(item(), 'bidder', 'https://sigma.midt.bg'); + expect(rss.title).toBe('Доставка на техника - Фирма ЕООД'); + expect(rss.link).toBe('https://sigma.midt.bg/contracts/e:UNP-1:2:eik:111111111'); + expect(rss.description).toContain('Изпълнител: Фирма ЕООД'); + expect(rss.description).toContain('Подписан: 15.05.2026'); + expect(rss.description).toContain('Процедура: Открита процедура'); + expect(rss.pubDate).toBe('Fri, 15 May 2026 00:00:00 GMT'); + }); + + it('builds a company-feed item around the buying authority', () => { + const rss = contractRssItem(item(), 'authority', 'https://sigma.midt.bg'); + expect(rss.title).toBe('Доставка на техника - Община Пример'); + expect(rss.description).toContain('Възложител: Община Пример'); + }); + + it('handles missing value and missing signing date', () => { + const rss = contractRssItem(item({ valueEur: null, signedAt: null }), 'bidder', 'https://x.bg'); + expect(rss.description).toContain('Стойност: без обявена стойност'); + expect(rss.description).not.toContain('Подписан:'); + expect(rss.pubDate).toBeNull(); + }); + + it('falls back to publishedAt for pubDate when there is no signing date', () => { + // Mirrors the query's ORDER BY COALESCE(signed_at, published_at): an item ordered by publish date + // must still carry a so readers can sort it. The description omits „Подписан:" (no signing). + const rss = contractRssItem( + item({ signedAt: null, publishedAt: '2026-05-10' }), + 'bidder', + 'https://x.bg', + ); + expect(rss.pubDate).toBe('Sun, 10 May 2026 00:00:00 GMT'); + expect(rss.description).not.toContain('Подписан:'); + }); +}); + +describe('rssFeed', () => { + const opts = { + title: 'Община <Пример> - нови договори', + description: 'Най-новите договори & анекси', + siteLink: 'https://sigma.midt.bg/authorities/123456789', + selfLink: 'https://sigma.midt.bg/authorities/123456789.rss', + items: [ + contractRssItem( + item({ subject: 'А/Б "проект" <спешен>' }), + 'bidder', + 'https://sigma.midt.bg', + ), + contractRssItem(item({ signedAt: null, valueEur: null }), 'bidder', 'https://sigma.midt.bg'), + ], + }; + + it('escapes user-controlled text everywhere it lands', () => { + const xml = rssFeed(opts); + expect(xml).toContain('Община <Пример> - нови договори'); + expect(xml).toContain('Най-новите договори & анекси'); + expect(xml).toContain('А/Б "проект" <спешен>'); + expect(xml).not.toMatch(/<спешен>/); + }); + + it('links the feed to itself and to the profile', () => { + const xml = rssFeed(opts); + expect(xml).toContain( + '', + ); + expect(xml).toContain('https://sigma.midt.bg/authorities/123456789'); + }); + + it('uses the contract URL as a permalink guid and skips pubDate for undated items', () => { + const xml = rssFeed(opts); + expect(xml).toContain( + 'https://sigma.midt.bg/contracts/e:UNP-1:2:eik:111111111', + ); + expect(xml.match(//g)).toHaveLength(2); // channel + the one dated item + }); + + it('stays deterministic: channel pubDate is the newest item date, no wall clock', () => { + const xml = rssFeed(opts); + expect(xml).toContain('Fri, 15 May 2026 00:00:00 GMT'); + expect(rssFeed(opts)).toBe(xml); + }); + + it('renders a valid empty channel when the entity has no contracts', () => { + const xml = rssFeed({ ...opts, items: [] }); + expect(xml).toContain(''); + expect(xml).not.toContain(''); + expect(xml).not.toContain(''); + }); + + it('channel pubDate is the MAX item date, not the first — even if items are mis-ordered', () => { + // rssFeed cannot enforce the caller's newest-first order, so it computes the max defensively. + const older = contractRssItem(item({ signedAt: '2026-01-10' }), 'bidder', 'https://x.bg'); + const newer = contractRssItem(item({ signedAt: '2026-08-20' }), 'bidder', 'https://x.bg'); + // Newest is SECOND in the array → a `find`-first would wrongly pick the older one. Anchor on the + // channel's line (only the channel pubDate follows it) to target the channel, not items. + const xml = rssFeed({ ...opts, items: [older, newer] }); + expect(xml).toContain( + 'bg\n Thu, 20 Aug 2026 00:00:00 GMT', + ); + }); +}); diff --git a/apps/web/app/lib/feed.ts b/apps/web/app/lib/feed.ts new file mode 100644 index 00000000..ff86fad4 --- /dev/null +++ b/apps/web/app/lib/feed.ts @@ -0,0 +1,121 @@ +import type { ContractListItem } from '@sigma/api-contract'; +import { date, money } from '@sigma/shared'; + +// RSS 2.0 for the entity profile feeds ("следи тази институция/фирма" without an account). +// Hand-rolled on purpose: the format is tiny, the itemset is capped at one page, and every value +// passes through xmlEscape - a templating dependency would be more surface than the format itself. + +const XML_ESCAPES: Record = { + '&': '&', + '<': '<', + '>': '>', + '"': '"', + "'": ''', +}; + +// Control characters that are NOT legal in XML 1.0 even when entity-escaped: everything below U+0020 +// except TAB (U+0009), LF (U+000A) and CR (U+000D). A stray one in a source subject/name would make +// the whole feed invalid XML and strict readers reject it, so drop them before escaping (review +// ydimitrof). There is no meaningful replacement — they carry no display value. +const XML_INVALID_CONTROL = /[\x00-\x08\x0B\x0C\x0E-\x1F]/g; + +export function xmlEscape(value: string): string { + return value.replace(XML_INVALID_CONTROL, '').replace(/[&<>"']/g, (ch) => XML_ESCAPES[ch] ?? ch); +} + +/** 'YYYY-MM-DD' -> RFC 822 (RSS pubDate); null for absent or malformed dates. */ +export function rssDate(day: string | null): string | null { + if (!day || !/^\d{4}-\d{2}-\d{2}$/.test(day)) return null; + const t = Date.parse(`${day}T00:00:00Z`); + return Number.isNaN(t) ? null : new Date(t).toUTCString(); +} + +export interface RssItem { + title: string; + /** Absolute URL; doubles as the permalink . */ + link: string; + description: string; + pubDate: string | null; +} + +/** + * One feed item per contract. `counterparty` picks the side the feed's reader does NOT follow: + * an authority feed lists winners ('bidder'), a company feed lists buyers ('authority'). + */ +export function contractRssItem( + item: ContractListItem, + counterparty: 'bidder' | 'authority', + origin: string, +): RssItem { + const other = counterparty === 'bidder' ? item.bidderDisplayName : item.authorityName; + const value = item.valueEur != null ? money(item.valueEur) : 'без обявена стойност'; + const parts = [ + `${counterparty === 'bidder' ? 'Изпълнител' : 'Възложител'}: ${other}`, + `Стойност: ${value}`, + item.signedAt ? `Подписан: ${date(item.signedAt)}` : null, + `Процедура: ${item.procedureLabel}`, + ].filter((p): p is string => p !== null); + return { + title: `${item.subject} - ${other}`, + link: `${origin}/contracts/${item.id}`, + description: parts.join(' · '), + // Fall back to publishedAt when there is no signing date, matching the query's + // `ORDER BY COALESCE(signed_at, published_at)`: an item positioned as "new" by publish date must + // carry a so readers can order it, and the channel pubDate stays the true newest (review + // ydimitrof). `??` (not `||`) is deliberate: it mirrors SQL COALESCE, which treats an empty string + // as PRESENT (only NULL falls through). Real data has NULL signed_at, so the two never disagree — + // and keeping the same rule here holds pubDate in sync with the row's ordering position. + pubDate: rssDate(item.signedAt ?? item.publishedAt), + }; +} + +export function rssFeed(opts: { + title: string; + description: string; + /** Absolute URL of the HTML profile the feed mirrors. */ + siteLink: string; + /** Absolute URL of the feed itself (atom:link rel="self"). */ + selfLink: string; + items: RssItem[]; +}): string { + const items = opts.items + .map((item) => + [ + ' ', + ` ${xmlEscape(item.title)}`, + ` ${xmlEscape(item.link)}`, + ` ${xmlEscape(item.link)}`, + item.pubDate ? ` ${xmlEscape(item.pubDate)}` : null, + ` ${xmlEscape(item.description)}`, + ' ', + ] + .filter((line): line is string => line !== null) + .join('\n'), + ) + .join('\n'); + // Channel-level pubDate is the newest item's date, computed as the MAX over all items rather than + // trusting input order: rssFeed is generic and cannot enforce the caller's newest-first ordering, so + // a future reordering must not silently yield a wrong channel (review ydimitrof). Still a + // pure function of the data — no "now" timestamp anywhere (RFC-822 dates compare via Date.parse). + const newest = opts.items.reduce((max, item) => { + if (!item.pubDate) return max; + return max === undefined || Date.parse(item.pubDate) > Date.parse(max) ? item.pubDate : max; + }, undefined); + return [ + '', + '', + ' ', + ` ${xmlEscape(opts.title)}`, + ` ${xmlEscape(opts.siteLink)}`, + ` ${xmlEscape(opts.description)}`, + ' bg', + newest ? ` ${xmlEscape(newest)}` : null, + ` `, + items || null, + ' ', + '', + '', + ] + .filter((line): line is string => line !== null) + .join('\n'); +} diff --git a/apps/web/app/routes.ts b/apps/web/app/routes.ts index 70909b7d..f8dad684 100644 --- a/apps/web/app/routes.ts +++ b/apps/web/app/routes.ts @@ -13,9 +13,11 @@ export default [ route('analytics', 'routes/analytics.tsx'), route('companies', 'routes/companies.tsx'), route('companies.csv', 'routes/companies.csv.tsx'), + route('companies/:eik.rss', 'routes/company.rss.tsx'), route('companies/:eik', 'routes/company.tsx'), route('authorities', 'routes/authorities.tsx'), route('authorities.csv', 'routes/authorities.csv.tsx'), + route('authorities/:eik.rss', 'routes/authority.rss.tsx'), route('authorities/:eik', 'routes/authority.tsx'), route('contracts', 'routes/contracts.tsx'), route('contracts.csv', 'routes/contracts.csv.tsx'), diff --git a/apps/web/app/routes/authority.rss.test.ts b/apps/web/app/routes/authority.rss.test.ts new file mode 100644 index 00000000..89a1d655 --- /dev/null +++ b/apps/web/app/routes/authority.rss.test.ts @@ -0,0 +1,103 @@ +import { describe, expect, it } from 'vitest'; +import { loader } from './authority.rss'; + +const contractRow = { + id: 'c:e:UNP-9:1:eik:222222222', + subject: 'Ремонт на път', + unp: 'UNP-9', + cpv_code: '45000000', + eu_funded: 0, + authority_id: 'auth:123456789', + authority_name: 'Община Пример', + bidder_id: 'eik:222222222', + bidder_name: 'Пътстрой ЕООД', + bidder_kind: 'company', + procedure_type: 'Открита процедура', + signed_at: '2026-05-15', + published_at: '2026-05-10', + bids_received: 2, + amount_eur: 1000, +}; + +function fakeDb(head: { name: string } | null, rows: unknown[] = []): D1Database { + return { + prepare(sql: string) { + const stmt = { + bind() { + return stmt; + }, + async first() { + return (sql.includes('authority_totals') ? head : null) as T; + }, + async all() { + return { results: rows as T[] }; + }, + }; + return stmt; + }, + } as unknown as D1Database; +} + +function call(url: string, eik: string, db: D1Database) { + return loader({ + request: new Request(url), + params: { eik }, + context: { cloudflare: { env: { DB: db } } }, + } as unknown as Parameters[0]); +} + +describe('authority.rss loader', () => { + it('serves an RSS feed with self/site links and bidder-side items', async () => { + const res = await call( + 'https://sigma.midt.bg/authorities/123456789.rss', + '123456789', + fakeDb({ name: 'Община Пример' }, [contractRow]), + ); + expect(res.status).toBe(200); + expect(res.headers.get('Content-Type')).toContain('application/rss+xml'); + expect(res.headers.get('X-Robots-Tag')).toBe('noindex'); + + const body = await res.text(); + expect(body).toContain('Община Пример - нови договори - СИГМА'); + expect(body).toContain('https://sigma.midt.bg/authorities/123456789'); + expect(body).toContain( + '', + ); + // An authority feed lists the WINNER (counterparty = 'bidder'). + expect(body).toContain('Изпълнител: Пътстрой ЕООД'); + // contractSlug strips the leading 'c:' from the id. + expect(body).toContain('https://sigma.midt.bg/contracts/e:UNP-9:1:eik:222222222'); + }); + + it('404s for an authority absent from the rollup', async () => { + const res = await call('https://sigma.midt.bg/authorities/999.rss', '999', fakeDb(null)); + expect(res.status).toBe(404); + }); + + it('404s gracefully for a garbage slug (no early format check, just a no-row lookup)', async () => { + // authorityIdFromSlug is total, so an invalid slug is not rejected up front — it resolves to an + // authority id that simply matches no row → getAuthorityHead null → 404 (review ydimitrof). + const res = await call( + 'https://sigma.midt.bg/authorities/not-an-eik.rss', + 'not-an-eik', + fakeDb(null), + ); + expect(res.status).toBe(404); + }); + + it('404s for an empty eik', async () => { + const res = await call('https://sigma.midt.bg/authorities/.rss', '.rss', fakeDb({ name: 'x' })); + expect(res.status).toBe(404); + }); + + it('strips a .rss suffix left in the param so the links are not doubled', async () => { + const res = await call( + 'https://sigma.midt.bg/authorities/123456789.rss', + '123456789.rss', + fakeDb({ name: 'Община Пример' }), + ); + const body = await res.text(); + expect(body).toContain('https://sigma.midt.bg/authorities/123456789'); + expect(body).not.toContain('123456789.rss.rss'); + }); +}); diff --git a/apps/web/app/routes/authority.rss.tsx b/apps/web/app/routes/authority.rss.tsx new file mode 100644 index 00000000..743b0db8 --- /dev/null +++ b/apps/web/app/routes/authority.rss.tsx @@ -0,0 +1,52 @@ +import { + authorityIdFromSlug, + authoritySlug, + getAuthorityHead, + listRecentEntityContracts, +} from '@sigma/db'; +import type { Route } from './+types/authority.rss'; +import { publicCache } from '../lib/cache'; +import { withDataSource } from '../lib/dataSource'; +import { contractRssItem, rssFeed } from '../lib/feed'; +import { withDbRetry } from '../lib/retry'; + +// Resource route: RSS 2.0 feed of an authority's newest contracts (/authorities/:eik.rss) - the +// no-account way to follow an entity (docs/api.md). X-Robots-Tag keeps feeds out of search indexes +// (profile pages carry the indexable content; some company profiles are deliberately noindex, #173). +export async function loader({ params, request, context }: Route.LoaderArgs) { + const eik = (params.eik ?? '').replace(/\.rss$/, ''); + if (!eik.trim()) return withDataSource(new Response('Not Found', { status: 404 })); + const db = context.cloudflare.env.DB; + // No early slug-validity 404 like company.rss (which short-circuits on bidderIdFromSlug === null): + // authorityIdFromSlug is total (authorities are always ЕИК-keyed, no fallible decode), and the raw + // ЕИК is not format-constrained in the pipeline, so validating here could reject a real profile and + // would diverge from the HTML authority route. An unknown slug just yields getAuthorityHead === null + // → 404 below, at the cost of one indexed no-row lookup (review ydimitrof). + const authorityId = authorityIdFromSlug(eik); + // Build the self/site links from the CANONICAL slug (derived from the resolved id), not the raw + // request param, so a resolvable-but-non-canonical request still emits the same URLs as the HTML + // profile (review ydimitrof). + const canonicalEik = authoritySlug(authorityId); + const { origin } = new URL(request.url); + return withDbRetry(async () => { + const head = await getAuthorityHead(db, authorityId); + if (!head) return withDataSource(new Response('Not Found', { status: 404 })); + const contracts = await listRecentEntityContracts(db, { kind: 'authority', authorityId }); + const xml = rssFeed({ + title: `${head.name} - нови договори - СИГМА`, + description: `Най-новите договори за обществени поръчки, възложени от ${head.name}.`, + siteLink: `${origin}/authorities/${canonicalEik}`, + selfLink: `${origin}/authorities/${canonicalEik}.rss`, + items: contracts.map((c) => contractRssItem(c, 'bidder', origin)), + }); + return withDataSource( + new Response(xml, { + headers: { + 'Content-Type': 'application/rss+xml; charset=utf-8', + 'Cache-Control': publicCache(3600), + 'X-Robots-Tag': 'noindex', + }, + }), + ); + }); +} diff --git a/apps/web/app/routes/authority.tsx b/apps/web/app/routes/authority.tsx index 20298c1b..7485f18d 100644 --- a/apps/web/app/routes/authority.tsx +++ b/apps/web/app/routes/authority.tsx @@ -348,9 +348,17 @@ export default function Authority({ loaderData }: Route.ComponentProps) { Виж всички / филтрирай / свали като CSV → + {' · '} + Следи новите договори (RSS)

+ ); } diff --git a/apps/web/app/routes/company.rss.test.ts b/apps/web/app/routes/company.rss.test.ts new file mode 100644 index 00000000..dc7a7d02 --- /dev/null +++ b/apps/web/app/routes/company.rss.test.ts @@ -0,0 +1,115 @@ +import { describe, expect, it } from 'vitest'; +import { loader } from './company.rss'; + +const contractRow = { + id: 'c:e:UNP-9:1:eik:222222222', + subject: 'Ремонт на път', + unp: 'UNP-9', + cpv_code: '45000000', + eu_funded: 0, + authority_id: 'auth:123456789', + authority_name: 'Община Пример', + bidder_id: 'eik:222222222', + bidder_name: 'Пътстрой ЕООД', + bidder_kind: 'company', + procedure_type: 'Открита процедура', + signed_at: '2026-05-15', + published_at: '2026-05-10', + bids_received: 2, + amount_eur: 1000, +}; + +function fakeDb(head: { name: string; kind: string } | null, rows: unknown[] = []): D1Database { + return { + prepare(sql: string) { + const stmt = { + bind() { + return stmt; + }, + async first() { + return (sql.includes('company_totals') ? head : null) as T; + }, + async all() { + return { results: rows as T[] }; + }, + }; + return stmt; + }, + } as unknown as D1Database; +} + +function call(url: string, eik: string, db: D1Database) { + return loader({ + request: new Request(url), + params: { eik }, + context: { cloudflare: { env: { DB: db } } }, + } as unknown as Parameters[0]); +} + +describe('company.rss loader', () => { + it('serves an RSS feed with self/site links and authority-side items', async () => { + const res = await call( + 'https://sigma.midt.bg/companies/222222222.rss', + '222222222', + fakeDb({ name: 'Пътстрой ЕООД', kind: 'company' }, [contractRow]), + ); + expect(res.status).toBe(200); + expect(res.headers.get('Content-Type')).toContain('application/rss+xml'); + expect(res.headers.get('X-Robots-Tag')).toBe('noindex'); + + const body = await res.text(); + expect(body).toContain('Пътстрой ЕООД - нови договори - СИГМА'); + expect(body).toContain('https://sigma.midt.bg/companies/222222222'); + expect(body).toContain( + '', + ); + // A company feed lists the BUYER (counterparty = 'authority'). + expect(body).toContain('Възложител: Община Пример'); + }); + + it('404s for a company absent from the rollup', async () => { + const res = await call( + 'https://sigma.midt.bg/companies/222222222.rss', + '222222222', + fakeDb(null), + ); + expect(res.status).toBe(404); + }); + + it('404s for an undecodable slug before touching the DB (no bidder id)', async () => { + // 'xyz' is neither a valid ЕИК nor an `n`-prefixed base64 name slug → bidderIdFromSlug returns null. + const res = await call( + 'https://sigma.midt.bg/companies/xyz.rss', + 'xyz', + fakeDb({ name: 'should not be read', kind: 'company' }), + ); + expect(res.status).toBe(404); + }); + + it('strips a .rss suffix left in the param so the links are not doubled', async () => { + const res = await call( + 'https://sigma.midt.bg/companies/222222222.rss', + '222222222.rss', + fakeDb({ name: 'Пътстрой ЕООД', kind: 'company' }), + ); + const body = await res.text(); + expect(body).toContain('https://sigma.midt.bg/companies/222222222'); + expect(body).not.toContain('222222222.rss.rss'); + }); + + it('builds self/site links from the canonical slug, not the raw request param', async () => { + // 'nWA==' is a non-canonical encoding of the name-keyed slug 'nWA' (companySlug strips the '=' + // base64 padding). The links must use the canonical 'nWA', matching the HTML profile. + const res = await call( + 'https://sigma.midt.bg/companies/nWA==.rss', + 'nWA==', + fakeDb({ name: 'X', kind: 'company' }), + ); + const body = await res.text(); + expect(body).toContain('https://sigma.midt.bg/companies/nWA'); + expect(body).toContain( + '', + ); + expect(body).not.toContain('nWA=='); + }); +}); diff --git a/apps/web/app/routes/company.rss.tsx b/apps/web/app/routes/company.rss.tsx new file mode 100644 index 00000000..4373f412 --- /dev/null +++ b/apps/web/app/routes/company.rss.tsx @@ -0,0 +1,48 @@ +import { + bidderIdFromSlug, + companySlug, + getCompanyHead, + listRecentEntityContracts, +} from '@sigma/db'; +import type { Route } from './+types/company.rss'; +import { publicCache } from '../lib/cache'; +import { withDataSource } from '../lib/dataSource'; +import { contractRssItem, rssFeed } from '../lib/feed'; +import { withDbRetry } from '../lib/retry'; + +// Resource route: RSS 2.0 feed of a company's newest contracts (/companies/:eik.rss) - the +// no-account way to follow an entity (docs/api.md). X-Robots-Tag keeps feeds out of search indexes +// (profile pages carry the indexable content; some company profiles are deliberately noindex, #173). +export async function loader({ params, request, context }: Route.LoaderArgs) { + const slug = (params.eik ?? '').replace(/\.rss$/, ''); + if (!slug.trim()) return withDataSource(new Response('Not Found', { status: 404 })); + const bidderId = bidderIdFromSlug(slug); + if (!bidderId) return withDataSource(new Response('Not Found', { status: 404 })); + // Build the self/site links from the CANONICAL slug (re-derived from the resolved bidder id), not the + // raw request param, so a resolvable-but-non-canonical request (e.g. a name-keyed slug with different + // base64 padding) still emits the same URLs as the HTML profile (review ydimitrof). + const canonicalSlug = companySlug(bidderId); + const db = context.cloudflare.env.DB; + const { origin } = new URL(request.url); + return withDbRetry(async () => { + const head = await getCompanyHead(db, bidderId); + if (!head) return withDataSource(new Response('Not Found', { status: 404 })); + const contracts = await listRecentEntityContracts(db, { kind: 'company', bidderId }); + const xml = rssFeed({ + title: `${head.name} - нови договори - СИГМА`, + description: `Най-новите договори за обществени поръчки, спечелени от ${head.name}.`, + siteLink: `${origin}/companies/${canonicalSlug}`, + selfLink: `${origin}/companies/${canonicalSlug}.rss`, + items: contracts.map((c) => contractRssItem(c, 'authority', origin)), + }); + return withDataSource( + new Response(xml, { + headers: { + 'Content-Type': 'application/rss+xml; charset=utf-8', + 'Cache-Control': publicCache(3600), + 'X-Robots-Tag': 'noindex', + }, + }), + ); + }); +} diff --git a/apps/web/app/routes/company.tsx b/apps/web/app/routes/company.tsx index 29caf1c8..4895204c 100644 --- a/apps/web/app/routes/company.tsx +++ b/apps/web/app/routes/company.tsx @@ -408,9 +408,17 @@ export default function Company({ loaderData }: Route.ComponentProps) { Виж всички / филтрирай / свали като CSV → + {' · '} + Следи новите договори (RSS)

+ ); } diff --git a/docs/api.md b/docs/api.md index f4bf0b24..9b18e776 100644 --- a/docs/api.md +++ b/docs/api.md @@ -54,6 +54,18 @@ HTML-страниците, всеки списък и всеки договор Това е единственият per-entity JSON днес; за институции/компании ползвайте CSV списъците или HTML профилите. +### RSS фийдове на профилите + +`GET /authorities/{ЕИК}.rss` и `GET /companies/{slug}.rss` → RSS 2.0 +(`application/rss+xml; charset=utf-8`) с най-новите до 50 договора на +институцията / компанията, подредени по дата на подписване (при липсваща дата - +по публикуване). Всеки запис носи предмета, насрещната страна, стойността и +процедурата; ``/`` водят към страницата на договора. Това е +първата стъпка на „наблюдаваните списъци": следене на субект без акаунт - от +RSS четец или автоматизация. 404 за непознат профил; фийдовете носят +`X-Robots-Tag: noindex` (индексируемото съдържание е HTML профилът, а +разпознатите ЕТ профили са умишлено noindex - вж. бележката за личните данни). + ### Sitemap-и `GET /sitemap.xml` (индекс) + `/sitemap-pages.xml`, `/sitemap-authorities.xml`, @@ -90,4 +102,5 @@ endpoint и **няма** обща REST заявка отвъд изброено Този документ покрива наличното днес. Ако ви трябва формат или endpoint, който липсва (напр. OCDS пакети, per-entity JSON за институции/компании, годишни -bulk dump-ове), отворете issue — посоката е координирана в `docs/`. +bulk dump-ове, email известия върху RSS фийдовете), отворете issue — посоката е +координирана в `docs/`. diff --git a/packages/api-contract/src/index.ts b/packages/api-contract/src/index.ts index 2adfeb29..1fff3e76 100644 --- a/packages/api-contract/src/index.ts +++ b/packages/api-contract/src/index.ts @@ -227,6 +227,7 @@ export interface ContractListItem { bidderKind: EntityKind; procedureLabel: string; signedAt: string | null; + publishedAt: string | null; // ordering/pubDate fallback when signedAt is absent (COALESCE order) bidsReceived: number | null; valueEur: number | null; // null = suspect / unconvertible → render the проверяват note } diff --git a/packages/db/migrations/0006_recent_feed_indexes.sql b/packages/db/migrations/0006_recent_feed_indexes.sql new file mode 100644 index 00000000..f035bee4 --- /dev/null +++ b/packages/db/migrations/0006_recent_feed_indexes.sql @@ -0,0 +1,23 @@ +-- Entity "newest contracts" feeds (RSS) paginate with +-- ORDER BY COALESCE(signed_at, published_at) DESC, id DESC LIMIT 50 +-- scoped to one company or one authority. Without a matching index SQLite gathers ALL of the entity's +-- contracts and does a full USE TEMP B-TREE FOR ORDER BY to return 50 — a public, unauthenticated +-- Denial-of-Wallet vector on a big supplier / ministry (D1 bills rows SCANNED). These two composite +-- indexes let each feed walk the index scoped to the entity and stop at LIMIT. + +-- Company feed: WHERE bidder_id = ? (already a column on contracts). +CREATE INDEX IF NOT EXISTS idx_contracts_bidder_recent + ON contracts(bidder_id, COALESCE(signed_at, published_at) DESC, id DESC); + +-- Authority feed: WHERE authority_id = ?. authority_id lives on `tenders`, not `contracts`, so a +-- scoped index is impossible without denormalising it onto the contract row. Add the column, backfill +-- it from the parent tender, and index it. The ETL keeps it populated (scripts/normalize-raw.sql and +-- scripts/refresh-slice.sql set it right after inserting contracts). SQLite ALTER ADD COLUMN has no +-- IF NOT EXISTS, but migrations apply once; the backfill covers rows that predate this migration. +ALTER TABLE contracts ADD COLUMN authority_id TEXT; +UPDATE contracts + SET authority_id = (SELECT t.authority_id FROM tenders t WHERE t.id = contracts.tender_id) + WHERE authority_id IS NULL; + +CREATE INDEX IF NOT EXISTS idx_contracts_authority_recent + ON contracts(authority_id, COALESCE(signed_at, published_at) DESC, id DESC); diff --git a/packages/db/src/queries/contracts.test.ts b/packages/db/src/queries/contracts.test.ts index 504754b2..117f0497 100644 --- a/packages/db/src/queries/contracts.test.ts +++ b/packages/db/src/queries/contracts.test.ts @@ -1,5 +1,10 @@ import { describe, expect, it } from 'vitest'; -import { getContractFacets, listContracts, normalizeContractSort } from './contracts'; +import { + getContractFacets, + listContracts, + listRecentEntityContracts, + normalizeContractSort, +} from './contracts'; describe('normalizeContractSort', () => { it('passes known sort keys through', () => { @@ -28,6 +33,7 @@ const contractRow = { bidder_kind: 'company' as const, procedure_type: 'Открита процедура', signed_at: '2024-01-01', + published_at: '2024-01-02', bids_received: 3, amount_eur: 1000, sort_value: 1000, @@ -128,3 +134,52 @@ describe('getContractFacets', () => { }); }); }); + +describe('listRecentEntityContracts', () => { + function capturingDb() { + const captured: { sql: string; binds: unknown[] }[] = []; + const db = { + prepare(sql: string) { + const entry = { sql, binds: [] as unknown[] }; + captured.push(entry); + return { + bind(...args: unknown[]) { + entry.binds = args; + return this; + }, + async all() { + return { results: [contractRow] as T[] }; + }, + }; + }, + } as D1Database; + return { db, captured }; + } + + it('scopes an authority feed by c.authority_id and orders date-desc with id tiebreak', async () => { + const { db, captured } = capturingDb(); + const items = await listRecentEntityContracts(db, { + kind: 'authority', + authorityId: 'auth:123456789', + }); + + expect(captured).toHaveLength(1); + // Scoped on the denormalised contract column (migration 0006), not t.authority_id, so it can use + // idx_contracts_authority_recent. + expect(captured[0]?.sql).toContain('WHERE c.authority_id = ?'); + expect(captured[0]?.sql).toContain( + 'ORDER BY COALESCE(c.signed_at, c.published_at) DESC, c.id DESC', + ); + expect(captured[0]?.binds).toEqual(['auth:123456789', 50]); + expect(items[0]?.subject).toBe('Subject'); + expect(items[0]?.id).toBe('1'); // contractSlug strips the 'c:' prefix + }); + + it('scopes a company feed by c.bidder_id and honours a custom limit', async () => { + const { db, captured } = capturingDb(); + await listRecentEntityContracts(db, { kind: 'company', bidderId: 'eik:111111111' }, 20); + + expect(captured[0]?.sql).toContain('WHERE c.bidder_id = ?'); + expect(captured[0]?.binds).toEqual(['eik:111111111', 20]); + }); +}); diff --git a/packages/db/src/queries/contracts.ts b/packages/db/src/queries/contracts.ts index e48d7b6d..a2bd7b5d 100644 --- a/packages/db/src/queries/contracts.ts +++ b/packages/db/src/queries/contracts.ts @@ -88,6 +88,7 @@ interface ContractRow { bidder_kind: 'company' | 'consortium'; procedure_type: string; signed_at: string | null; + published_at: string | null; bids_received: number | null; amount_eur: number | null; } @@ -96,7 +97,7 @@ const SELECT = ` SELECT c.id, COALESCE(NULLIF(c.contract_subject, ''), t.title) AS subject, t.source_id AS unp, t.cpv_code, c.eu_funded, t.authority_id, a.name AS authority_name, c.bidder_id, b.name AS bidder_name, b.kind AS bidder_kind, - t.procedure_type, c.signed_at, c.bids_received, c.amount_eur`; + t.procedure_type, c.signed_at, c.published_at, c.bids_received, c.amount_eur`; const FROM = ` FROM contracts c JOIN tenders t ON t.id = c.tender_id @@ -208,6 +209,7 @@ function toItem(r: ContractRow): ContractListItem { bidderKind: r.bidder_kind, procedureLabel: procedureGroup(r.procedure_type).label, signedAt: r.signed_at, + publishedAt: r.published_at, bidsReceived: r.bids_received, valueEur: r.amount_eur, }; @@ -235,6 +237,30 @@ export async function listSingleOfferContracts( return rows.results.map(toItem); } +/** + * The newest contracts of one entity, for the profile RSS feeds - date-desc with the same + * signed/published fallback the profile "Най-нови" tab uses, id tiebreak for a stable order. + * Reuses the shared SELECT/FROM and row mapper so feed items match the HTML lists exactly. + */ +export async function listRecentEntityContracts( + db: D1Database, + entity: { kind: 'authority'; authorityId: string } | { kind: 'company'; bidderId: string }, + limit = 50, +): Promise { + // Scope on the contract-row columns (bidder_id, and the denormalised authority_id — migration 0006) + // so the ORDER BY walks idx_contracts_{bidder,authority}_recent scoped to the entity and stops at + // LIMIT, instead of gathering all of the entity's contracts and temp-B-tree-sorting them (DoW). + const scope = entity.kind === 'authority' ? 'c.authority_id = ?' : 'c.bidder_id = ?'; + const id = entity.kind === 'authority' ? entity.authorityId : entity.bidderId; + const rows = await db + .prepare( + `${SELECT} ${FROM} WHERE ${scope} ORDER BY COALESCE(c.signed_at, c.published_at) DESC, c.id DESC LIMIT ?`, + ) + .bind(id, limit) + .all(); + return rows.results.map(toItem); +} + export interface ContractListResult extends Page { valueEur: number; suspect: number; diff --git a/packages/db/src/queries/details.ts b/packages/db/src/queries/details.ts index e4ac70be..e5205df9 100644 --- a/packages/db/src/queries/details.ts +++ b/packages/db/src/queries/details.ts @@ -93,6 +93,33 @@ interface CompanyTotalsFull { last_date: string | null; } +/** + * Display name for an authority's RSS feed header - one indexed rollup read instead of the full + * profile DTO. Null mirrors the profile page's 404 (an entity absent from authority_totals). + */ +export async function getAuthorityHead( + db: D1Database, + authorityId: string, +): Promise<{ name: string } | null> { + const row = await db + .prepare(`SELECT name FROM authority_totals WHERE authority_id = ?`) + .bind(authorityId) + .first<{ name: string }>(); + return row ? { name: cleanName(row.name) } : null; +} + +/** Company counterpart of getAuthorityHead - same display-name rules as the profile page. */ +export async function getCompanyHead( + db: D1Database, + bidderId: string, +): Promise<{ name: string } | null> { + const row = await db + .prepare(`SELECT name, kind FROM company_totals WHERE bidder_id = ?`) + .bind(bidderId) + .first<{ name: string; kind: 'company' | 'consortium' }>(); + return row ? { name: entityName(cleanName(row.name), row.kind) } : null; +} + export async function getCompany(db: D1Database, bidderId: string): Promise { const row = await db .prepare(`SELECT * FROM company_totals WHERE bidder_id = ?`) diff --git a/packages/db/src/recent-contracts-schema.test.ts b/packages/db/src/recent-contracts-schema.test.ts new file mode 100644 index 00000000..ba93764a --- /dev/null +++ b/packages/db/src/recent-contracts-schema.test.ts @@ -0,0 +1,63 @@ +/// +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +// listRecentEntityContracts (queries/contracts.ts) selects `c.published_at` and orders by +// COALESCE(c.signed_at, c.published_at). `published_at` exists on BOTH `contracts` and `tenders`, so a +// wrong alias (`t.published_at`) would NOT error — it would silently order by the tender's publish date +// instead of the contract's, and the mock-D1 unit tests can't catch that. This runs the real SELECT +// shape against the real migrated schema, with the contract's and tender's published_at deliberately +// different, to prove the query reads the CONTRACT column (review ydimitrof). + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); +const migration0 = resolve(root, 'packages/db/migrations/0000_init.sql'); + +function sqliteJson(dbPath: string, sql: string): T[] { + const out = execFileSync('sqlite3', ['-json', dbPath, sql], { encoding: 'utf8' }).trim(); + return out ? (JSON.parse(out) as T[]) : []; +} + +describe('listRecentEntityContracts column/alias against the real schema', () => { + let dir: string; + let dbPath: string; + + beforeAll(() => { + dir = mkdtempSync(resolve(tmpdir(), 'sigma-recent-')); + dbPath = resolve(dir, 'test.sqlite'); + execFileSync('sqlite3', ['-bail', dbPath], { input: `.read ${migration0}\n`, stdio: 'pipe' }); + execFileSync('sqlite3', ['-bail', dbPath], { + input: [ + `INSERT INTO authorities(id,name) VALUES('auth:1','A');`, + `INSERT INTO bidders(id,name) VALUES('eik:1','B');`, + // Tender publish date is DELIBERATELY different from the contract's. + `INSERT INTO tenders(id,source_id,title,authority_id,cpv_code,procedure_type,status,published_at) ` + + `VALUES('t:1','U1','T','auth:1','45000000','открита','awarded','2020-01-01');`, + // No signing date → the ORDER BY falls back to the contract's published_at. + `INSERT INTO contracts(id,tender_id,bidder_id,amount,amount_eur,currency,value_flag,signed_at,published_at,bids_received) ` + + `VALUES('c:1','t:1','eik:1',1000,1000,'EUR','ok',NULL,'2024-09-09',1);`, + ].join('\n'), + stdio: 'pipe', + }); + }); + + afterAll(() => rmSync(dir, { recursive: true, force: true })); + + it('reads c.published_at (the contract column), distinct from t.published_at', () => { + const rows = sqliteJson<{ cpub: string; tpub: string; ord: string }>( + dbPath, + `SELECT c.published_at AS cpub, t.published_at AS tpub, + COALESCE(c.signed_at, c.published_at) AS ord + FROM contracts c JOIN tenders t ON t.id = c.tender_id + WHERE t.authority_id = 'auth:1'`, + ); + expect(rows).toHaveLength(1); + expect(rows[0]?.cpub).toBe('2024-09-09'); // contract's own publish date + expect(rows[0]?.tpub).toBe('2020-01-01'); // the tender's — proves the two columns differ + // The recency fallback used by listRecentEntityContracts must resolve to the CONTRACT's date. + expect(rows[0]?.ord).toBe('2024-09-09'); + }); +}); diff --git a/packages/db/src/recent-feed-index.test.ts b/packages/db/src/recent-feed-index.test.ts new file mode 100644 index 00000000..41e84afb --- /dev/null +++ b/packages/db/src/recent-feed-index.test.ts @@ -0,0 +1,112 @@ +/// +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, readdirSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +// The entity "newest contracts" RSS feeds (listRecentEntityContracts) run, scoped to one company or +// authority, `ORDER BY COALESCE(signed_at, published_at) DESC, id DESC LIMIT 50`. Without a matching +// scoped index the planner gathers ALL of the entity's contracts and does USE TEMP B-TREE FOR ORDER +// BY — a public, unauthenticated Denial-of-Wallet on a big supplier/ministry (D1 bills rows scanned). +// Migration 0006 adds the composite indexes (and denormalises authority_id onto contracts so the +// authority scope is a contract-row column). This proves, on a real sqlite3 without ANALYZE (matching +// D1), that BEFORE the migration both feeds temp-B-tree-sort, and AFTER each walks its scoped index. + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); +const migrationsDir = resolve(root, 'packages/db/migrations'); +const feedMigration = readdirSync(migrationsDir).find((f) => f.includes('recent_feed_indexes')); +if (!feedMigration) throw new Error('recent_feed_indexes migration not found'); +const baseMigrations = readdirSync(migrationsDir) + .filter((f) => f.endsWith('.sql') && f !== feedMigration) + .sort(); + +function readScript(dbPath: string, file: string): void { + execFileSync('sqlite3', ['-bail', dbPath], { + input: `.read ${resolve(migrationsDir, file)}\n`, + stdio: 'pipe', + }); +} + +function exec(dbPath: string, sql: string): void { + execFileSync('sqlite3', ['-bail', dbPath], { input: sql, stdio: 'pipe' }); +} + +function plan(dbPath: string, sql: string): string { + return execFileSync('sqlite3', [dbPath], { + input: `EXPLAIN QUERY PLAN ${sql}\n`, + encoding: 'utf8', + }); +} + +const FROM = + 'FROM contracts c JOIN tenders t ON t.id = c.tender_id ' + + 'JOIN authorities a ON a.id = t.authority_id JOIN bidders b ON b.id = c.bidder_id'; +const ORDER = 'ORDER BY COALESCE(c.signed_at, c.published_at) DESC, c.id DESC LIMIT 50'; + +const companyFeed = `SELECT c.id ${FROM} WHERE c.bidder_id = 'eik:0' ${ORDER}`; +// BEFORE denormalisation the authority feed had to scope via the tender (no authority_id on contracts). +const authorityFeedOld = `SELECT c.id ${FROM} WHERE t.authority_id = 'auth:0' ${ORDER}`; +// AFTER: scoped on the denormalised contract column, so idx_contracts_authority_recent applies. +const authorityFeedNew = `SELECT c.id ${FROM} WHERE c.authority_id = 'auth:0' ${ORDER}`; + +function seed(dbPath: string, withAuthorityId: boolean): void { + const stmts: string[] = ['BEGIN;']; + for (let i = 0; i < 20; i++) + stmts.push(`INSERT INTO authorities(id,name) VALUES('auth:${i}','A${i}');`); + for (let i = 0; i < 30; i++) + stmts.push(`INSERT INTO bidders(id,name) VALUES('eik:${i}','B${i}');`); + for (let i = 0; i < 100; i++) + stmts.push( + `INSERT INTO tenders(id,source_id,title,authority_id,cpv_code,procedure_type,status) ` + + `VALUES('t:${i}','U${i}','T${i}','auth:${i % 20}','45000000','открита','awarded');`, + ); + for (let i = 0; i < 800; i++) + stmts.push( + `INSERT INTO contracts(id,tender_id,bidder_id,amount,amount_eur,currency,value_flag,signed_at,published_at,bids_received) ` + + `VALUES('c:${i}','t:${i % 100}','eik:${i % 30}',1,1,'EUR','ok','202${i % 5}-0${(i % 9) + 1}-15','202${i % 5}-0${(i % 9) + 1}-10',1);`, + ); + stmts.push('COMMIT;'); + exec(dbPath, stmts.join('\n')); + // Mirror the ETL: populate the denormalised authority_id from the parent tender (migration 0006 does + // the same via its backfill + the UPDATE steps in normalize-raw.sql / refresh-slice.sql). + if (withAuthorityId) + exec( + dbPath, + `UPDATE contracts SET authority_id = (SELECT t.authority_id FROM tenders t WHERE t.id = contracts.tender_id);`, + ); +} + +describe('entity recent-contracts feed indexes', () => { + let dir: string; + let before: string; + let after: string; + + beforeAll(() => { + dir = mkdtempSync(resolve(tmpdir(), 'sigma-recent-feed-')); + before = resolve(dir, 'before.sqlite'); + after = resolve(dir, 'after.sqlite'); + for (const m of baseMigrations) readScript(before, m); + seed(before, false); + for (const m of baseMigrations) readScript(after, m); + readScript(after, feedMigration); + seed(after, true); + }); + + afterAll(() => rmSync(dir, { recursive: true, force: true })); + + it('company feed full-scans + temp-sorts BEFORE, index-walks AFTER', () => { + expect(plan(before, companyFeed)).toContain('USE TEMP B-TREE FOR ORDER BY'); + const p = plan(after, companyFeed); + expect(p).toContain('idx_contracts_bidder_recent'); + expect(p).not.toContain('USE TEMP B-TREE FOR ORDER BY'); + }); + + it('authority feed full-scans + temp-sorts BEFORE, index-walks AFTER (via denormalised authority_id)', () => { + expect(plan(before, authorityFeedOld)).toContain('USE TEMP B-TREE FOR ORDER BY'); + const p = plan(after, authorityFeedNew); + expect(p).toContain('idx_contracts_authority_recent'); + expect(p).not.toContain('USE TEMP B-TREE FOR ORDER BY'); + }); +}); diff --git a/packages/db/src/refresh-slice.test.ts b/packages/db/src/refresh-slice.test.ts index aa20ec76..733dfb9f 100644 --- a/packages/db/src/refresh-slice.test.ts +++ b/packages/db/src/refresh-slice.test.ts @@ -1,6 +1,6 @@ /// import { execFileSync } from 'node:child_process'; -import { mkdtempSync, rmSync } from 'node:fs'; +import { mkdtempSync, readdirSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { dirname, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -8,7 +8,9 @@ import { describe, expect, it } from 'vitest'; import { assertIntegrity } from '../../../scripts/integrity-checks.mjs'; const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); -const schemaPath = resolve(root, 'packages/db/migrations/0000_init.sql'); +// Apply the whole migration chain (not just 0000): refresh-slice.sql/normalize-raw.sql now touch the +// denormalised contracts.authority_id column added by migration 0006, so the served schema must match. +const migrationsDir = resolve(root, 'packages/db/migrations'); const refreshSlicePath = resolve(root, 'scripts/refresh-slice.sql'); const normalizePath = resolve(root, 'scripts/normalize-raw.sql'); const workStagingSchemaPath = resolve(root, 'scripts/work-staging-schema.sql'); @@ -35,6 +37,15 @@ function readScript(dbPath: string, path: string): void { }); } +// Build the served schema from the full migration chain, as production does. +function applyMigrations(dbPath: string): void { + for (const f of readdirSync(migrationsDir) + .filter((f) => f.endsWith('.sql')) + .sort()) { + readScript(dbPath, resolve(migrationsDir, f)); + } +} + function resetRawStaging(dbPath: string): void { const rows = sqliteJson<{ name: string }>( dbPath, @@ -174,7 +185,7 @@ function seedOcdsOnlySharedNumber(dbPath: string): void { } function initWorkDb(dbPath: string): void { - readScript(dbPath, schemaPath); + applyMigrations(dbPath); readScript(dbPath, workStagingSchemaPath); } @@ -357,7 +368,7 @@ describe('refresh-slice EOP base derivation', () => { const dir = mkdtempSync(resolve(tmpdir(), 'sigma-refresh-slice-')); const dbPath = resolve(dir, 'test.sqlite'); try { - readScript(dbPath, schemaPath); + applyMigrations(dbPath); readScript(dbPath, workStagingSchemaPath); seedEopBaseDay(dbPath); @@ -436,7 +447,7 @@ describe('refresh-slice EOP base derivation', () => { const dir = mkdtempSync(resolve(tmpdir(), 'sigma-refresh-slice-')); const dbPath = resolve(dir, 'test.sqlite'); try { - readScript(dbPath, schemaPath); + applyMigrations(dbPath); readScript(dbPath, workStagingSchemaPath); seedEopOnlySharedNumber(dbPath); readScript(dbPath, refreshSlicePath); @@ -484,7 +495,7 @@ describe('refresh-slice EOP base derivation', () => { const dir = mkdtempSync(resolve(tmpdir(), 'sigma-refresh-slice-')); const dbPath = resolve(dir, 'test.sqlite'); try { - readScript(dbPath, schemaPath); + applyMigrations(dbPath); readScript(dbPath, workStagingSchemaPath); sqlite( dbPath, @@ -532,7 +543,7 @@ describe('refresh-slice EOP base derivation', () => { const dir = mkdtempSync(resolve(tmpdir(), 'sigma-refresh-slice-')); const dbPath = resolve(dir, 'test.sqlite'); try { - readScript(dbPath, schemaPath); + applyMigrations(dbPath); readScript(dbPath, workStagingSchemaPath); sqlite( dbPath, diff --git a/scripts/normalize-raw.sql b/scripts/normalize-raw.sql index 17cad66e..9610ade7 100644 --- a/scripts/normalize-raw.sql +++ b/scripts/normalize-raw.sql @@ -560,6 +560,13 @@ WHERE x.bidder_key IS NOT NULL AND EXISTS (SELECT 1 FROM tenders te WHERE te.id = 't:' || x.unp) AND EXISTS (SELECT 1 FROM bidders b WHERE b.id = x.bidder_key); +-- Denormalise the parent tender's authority onto the contract row so the entity RSS feed can walk a +-- scoped index (idx_contracts_authority_recent, migration 0006) instead of temp-B-tree-sorting all of +-- an authority's contracts. tender_id is set above and tenders.id is the PK, so this is an indexed lookup. +UPDATE contracts + SET authority_id = (SELECT t.authority_id FROM tenders t WHERE t.id = contracts.tender_id) + WHERE authority_id IS NULL; + -- Reconciliation guard: the final summary reports the contracts inserted alongside the surviving -- staging candidates, so a future NOT NULL/foreign-key mismatch is visible instead of hidden by -- INSERT OR IGNORE. diff --git a/scripts/refresh-slice.sql b/scripts/refresh-slice.sql index e051a7ed..a08a184a 100644 --- a/scripts/refresh-slice.sql +++ b/scripts/refresh-slice.sql @@ -1002,6 +1002,14 @@ SET status = 'awarded' WHERE status <> 'awarded' AND EXISTS (SELECT 1 FROM raw_contracts c WHERE 't:' || c.unp = tenders.id); +-- Denormalise authority onto the touched (window) contracts so the authority RSS feed's scoped index +-- (idx_contracts_authority_recent, migration 0006) covers the just-inserted rows too — otherwise a +-- newly-signed contract would be missing from the feed until the next full import. Scoped to the +-- touched set; tenders.id is the PK, so the lookup is indexed. +UPDATE contracts + SET authority_id = (SELECT t.authority_id FROM tenders t WHERE t.id = contracts.tender_id) + WHERE id IN (SELECT id FROM refresh_touched_contracts); + -- 5) Promote window amendments into served domain history and roll touched contracts. -- @refresh-batch amendments