From edacf32b56f086cd28539655a19c5a487b6ce548 Mon Sep 17 00:00:00 2001 From: Rumen Slavov <26761822+B353N@users.noreply.github.com> Date: Sat, 4 Jul 2026 11:24:51 +0300 Subject: [PATCH 1/9] feat(db): recent-contracts query and profile head lookups for feeds --- packages/db/src/queries/contracts.test.ts | 54 ++++++++++++++++++++++- packages/db/src/queries/contracts.ts | 21 +++++++++ packages/db/src/queries/details.ts | 27 ++++++++++++ 3 files changed, 101 insertions(+), 1 deletion(-) diff --git a/packages/db/src/queries/contracts.test.ts b/packages/db/src/queries/contracts.test.ts index 504754b2..810dc079 100644 --- a/packages/db/src/queries/contracts.test.ts +++ b/packages/db/src/queries/contracts.test.ts @@ -1,5 +1,10 @@ import { describe, expect, it } from 'vitest'; -import { getContractFacets, listContracts, normalizeContractSort } from './contracts'; +import { + getContractFacets, + listContracts, + listRecentEntityContracts, + normalizeContractSort, +} from './contracts'; describe('normalizeContractSort', () => { it('passes known sort keys through', () => { @@ -128,3 +133,50 @@ describe('getContractFacets', () => { }); }); }); + +describe('listRecentEntityContracts', () => { + function capturingDb() { + const captured: { sql: string; binds: unknown[] }[] = []; + const db = { + prepare(sql: string) { + const entry = { sql, binds: [] as unknown[] }; + captured.push(entry); + return { + bind(...args: unknown[]) { + entry.binds = args; + return this; + }, + async all() { + return { results: [contractRow] as T[] }; + }, + }; + }, + } as D1Database; + return { db, captured }; + } + + it('scopes an authority feed by t.authority_id and orders date-desc with id tiebreak', async () => { + const { db, captured } = capturingDb(); + const items = await listRecentEntityContracts(db, { + kind: 'authority', + authorityId: 'auth:123456789', + }); + + expect(captured).toHaveLength(1); + expect(captured[0]?.sql).toContain('WHERE t.authority_id = ?'); + expect(captured[0]?.sql).toContain( + 'ORDER BY COALESCE(c.signed_at, c.published_at) DESC, c.id DESC', + ); + expect(captured[0]?.binds).toEqual(['auth:123456789', 50]); + expect(items[0]?.subject).toBe('Subject'); + expect(items[0]?.id).toBe('1'); // contractSlug strips the 'c:' prefix + }); + + it('scopes a company feed by c.bidder_id and honours a custom limit', async () => { + const { db, captured } = capturingDb(); + await listRecentEntityContracts(db, { kind: 'company', bidderId: 'eik:111111111' }, 20); + + expect(captured[0]?.sql).toContain('WHERE c.bidder_id = ?'); + expect(captured[0]?.binds).toEqual(['eik:111111111', 20]); + }); +}); diff --git a/packages/db/src/queries/contracts.ts b/packages/db/src/queries/contracts.ts index e48d7b6d..10b0b825 100644 --- a/packages/db/src/queries/contracts.ts +++ b/packages/db/src/queries/contracts.ts @@ -235,6 +235,27 @@ export async function listSingleOfferContracts( return rows.results.map(toItem); } +/** + * The newest contracts of one entity, for the profile RSS feeds - date-desc with the same + * signed/published fallback the profile "Най-нови" tab uses, id tiebreak for a stable order. + * Reuses the shared SELECT/FROM and row mapper so feed items match the HTML lists exactly. + */ +export async function listRecentEntityContracts( + db: D1Database, + entity: { kind: 'authority'; authorityId: string } | { kind: 'company'; bidderId: string }, + limit = 50, +): Promise { + const scope = entity.kind === 'authority' ? 't.authority_id = ?' : 'c.bidder_id = ?'; + const id = entity.kind === 'authority' ? entity.authorityId : entity.bidderId; + const rows = await db + .prepare( + `${SELECT} ${FROM} WHERE ${scope} ORDER BY COALESCE(c.signed_at, c.published_at) DESC, c.id DESC LIMIT ?`, + ) + .bind(id, limit) + .all(); + return rows.results.map(toItem); +} + export interface ContractListResult extends Page { valueEur: number; suspect: number; diff --git a/packages/db/src/queries/details.ts b/packages/db/src/queries/details.ts index e4ac70be..e5205df9 100644 --- a/packages/db/src/queries/details.ts +++ b/packages/db/src/queries/details.ts @@ -93,6 +93,33 @@ interface CompanyTotalsFull { last_date: string | null; } +/** + * Display name for an authority's RSS feed header - one indexed rollup read instead of the full + * profile DTO. Null mirrors the profile page's 404 (an entity absent from authority_totals). + */ +export async function getAuthorityHead( + db: D1Database, + authorityId: string, +): Promise<{ name: string } | null> { + const row = await db + .prepare(`SELECT name FROM authority_totals WHERE authority_id = ?`) + .bind(authorityId) + .first<{ name: string }>(); + return row ? { name: cleanName(row.name) } : null; +} + +/** Company counterpart of getAuthorityHead - same display-name rules as the profile page. */ +export async function getCompanyHead( + db: D1Database, + bidderId: string, +): Promise<{ name: string } | null> { + const row = await db + .prepare(`SELECT name, kind FROM company_totals WHERE bidder_id = ?`) + .bind(bidderId) + .first<{ name: string; kind: 'company' | 'consortium' }>(); + return row ? { name: entityName(cleanName(row.name), row.kind) } : null; +} + export async function getCompany(db: D1Database, bidderId: string): Promise { const row = await db .prepare(`SELECT * FROM company_totals WHERE bidder_id = ?`) From 083f1b938d6be8b19c8a471d1251ca774078dd43 Mon Sep 17 00:00:00 2001 From: Rumen Slavov <26761822+B353N@users.noreply.github.com> Date: Sat, 4 Jul 2026 11:24:51 +0300 Subject: [PATCH 2/9] feat(web): rss feeds with the newest contracts on entity profiles --- apps/web/app/lib/feed.test.ts | 125 ++++++++++++++++++++++++++ apps/web/app/lib/feed.ts | 104 +++++++++++++++++++++ apps/web/app/routes.ts | 2 + apps/web/app/routes/authority.rss.tsx | 38 ++++++++ apps/web/app/routes/authority.tsx | 8 ++ apps/web/app/routes/company.rss.tsx | 39 ++++++++ apps/web/app/routes/company.tsx | 8 ++ 7 files changed, 324 insertions(+) create mode 100644 apps/web/app/lib/feed.test.ts create mode 100644 apps/web/app/lib/feed.ts create mode 100644 apps/web/app/routes/authority.rss.tsx create mode 100644 apps/web/app/routes/company.rss.tsx diff --git a/apps/web/app/lib/feed.test.ts b/apps/web/app/lib/feed.test.ts new file mode 100644 index 00000000..e958de18 --- /dev/null +++ b/apps/web/app/lib/feed.test.ts @@ -0,0 +1,125 @@ +import { describe, expect, it } from 'vitest'; +import type { ContractListItem } from '@sigma/api-contract'; +import { contractRssItem, rssDate, rssFeed, xmlEscape } from './feed'; + +function item(overrides: Partial = {}): ContractListItem { + return { + id: 'e:UNP-1:2:eik:111111111', + subject: 'Доставка на техника', + unp: 'UNP-1', + sectorCode: '30', + euFunded: false, + isConsortium: false, + authoritySlug: '123456789', + authorityName: 'Община Пример', + bidderSlug: '111111111', + bidderName: 'Фирма ЕООД', + bidderDisplayName: 'Фирма ЕООД', + bidderKind: 'company', + procedureLabel: 'Открита процедура', + signedAt: '2026-05-15', + bidsReceived: 3, + valueEur: 12345.67, + ...overrides, + }; +} + +describe('xmlEscape', () => { + it('escapes the five XML special characters', () => { + expect(xmlEscape(``)).toBe('<a & "b" & 'c'>'); + }); + it('passes ordinary Bulgarian text through unchanged', () => { + expect(xmlEscape('Община „Пример“ - договори')).toBe('Община „Пример“ - договори'); + }); +}); + +describe('rssDate', () => { + it('renders an ISO day as RFC 822', () => { + expect(rssDate('2026-05-15')).toBe('Fri, 15 May 2026 00:00:00 GMT'); + }); + it('returns null for null, malformed, and impossible dates', () => { + expect(rssDate(null)).toBeNull(); + expect(rssDate('')).toBeNull(); + expect(rssDate('15.05.2026')).toBeNull(); + expect(rssDate('2026-13-45')).toBeNull(); + }); +}); + +describe('contractRssItem', () => { + it('builds an authority-feed item around the winning bidder', () => { + const rss = contractRssItem(item(), 'bidder', 'https://sigma.midt.bg'); + expect(rss.title).toBe('Доставка на техника - Фирма ЕООД'); + expect(rss.link).toBe('https://sigma.midt.bg/contracts/e:UNP-1:2:eik:111111111'); + expect(rss.description).toContain('Изпълнител: Фирма ЕООД'); + expect(rss.description).toContain('Подписан: 15.05.2026'); + expect(rss.description).toContain('Процедура: Открита процедура'); + expect(rss.pubDate).toBe('Fri, 15 May 2026 00:00:00 GMT'); + }); + + it('builds a company-feed item around the buying authority', () => { + const rss = contractRssItem(item(), 'authority', 'https://sigma.midt.bg'); + expect(rss.title).toBe('Доставка на техника - Община Пример'); + expect(rss.description).toContain('Възложител: Община Пример'); + }); + + it('handles missing value and missing signing date', () => { + const rss = contractRssItem(item({ valueEur: null, signedAt: null }), 'bidder', 'https://x.bg'); + expect(rss.description).toContain('Стойност: без обявена стойност'); + expect(rss.description).not.toContain('Подписан:'); + expect(rss.pubDate).toBeNull(); + }); +}); + +describe('rssFeed', () => { + const opts = { + title: 'Община <Пример> - нови договори', + description: 'Най-новите договори & анекси', + siteLink: 'https://sigma.midt.bg/authorities/123456789', + selfLink: 'https://sigma.midt.bg/authorities/123456789.rss', + items: [ + contractRssItem( + item({ subject: 'А/Б "проект" <спешен>' }), + 'bidder', + 'https://sigma.midt.bg', + ), + contractRssItem(item({ signedAt: null, valueEur: null }), 'bidder', 'https://sigma.midt.bg'), + ], + }; + + it('escapes user-controlled text everywhere it lands', () => { + const xml = rssFeed(opts); + expect(xml).toContain('Община <Пример> - нови договори'); + expect(xml).toContain('Най-новите договори & анекси'); + expect(xml).toContain('А/Б "проект" <спешен>'); + expect(xml).not.toMatch(/<спешен>/); + }); + + it('links the feed to itself and to the profile', () => { + const xml = rssFeed(opts); + expect(xml).toContain( + '', + ); + expect(xml).toContain('https://sigma.midt.bg/authorities/123456789'); + }); + + it('uses the contract URL as a permalink guid and skips pubDate for undated items', () => { + const xml = rssFeed(opts); + expect(xml).toContain( + 'https://sigma.midt.bg/contracts/e:UNP-1:2:eik:111111111', + ); + expect(xml.match(//g)).toHaveLength(2); // channel + the one dated item + }); + + it('stays deterministic: channel pubDate is the newest item date, no wall clock', () => { + const xml = rssFeed(opts); + expect(xml).toContain('Fri, 15 May 2026 00:00:00 GMT'); + expect(rssFeed(opts)).toBe(xml); + }); + + it('renders a valid empty channel when the entity has no contracts', () => { + const xml = rssFeed({ ...opts, items: [] }); + expect(xml).toContain(''); + expect(xml).not.toContain(''); + expect(xml).not.toContain(''); + }); +}); diff --git a/apps/web/app/lib/feed.ts b/apps/web/app/lib/feed.ts new file mode 100644 index 00000000..fbe26a7d --- /dev/null +++ b/apps/web/app/lib/feed.ts @@ -0,0 +1,104 @@ +import type { ContractListItem } from '@sigma/api-contract'; +import { date, money } from '@sigma/shared'; + +// RSS 2.0 for the entity profile feeds ("следи тази институция/фирма" without an account). +// Hand-rolled on purpose: the format is tiny, the itemset is capped at one page, and every value +// passes through xmlEscape - a templating dependency would be more surface than the format itself. + +const XML_ESCAPES: Record = { + '&': '&', + '<': '<', + '>': '>', + '"': '"', + "'": ''', +}; + +export function xmlEscape(value: string): string { + return value.replace(/[&<>"']/g, (ch) => XML_ESCAPES[ch] ?? ch); +} + +/** 'YYYY-MM-DD' -> RFC 822 (RSS pubDate); null for absent or malformed dates. */ +export function rssDate(day: string | null): string | null { + if (!day || !/^\d{4}-\d{2}-\d{2}$/.test(day)) return null; + const t = Date.parse(`${day}T00:00:00Z`); + return Number.isNaN(t) ? null : new Date(t).toUTCString(); +} + +export interface RssItem { + title: string; + /** Absolute URL; doubles as the permalink . */ + link: string; + description: string; + pubDate: string | null; +} + +/** + * One feed item per contract. `counterparty` picks the side the feed's reader does NOT follow: + * an authority feed lists winners ('bidder'), a company feed lists buyers ('authority'). + */ +export function contractRssItem( + item: ContractListItem, + counterparty: 'bidder' | 'authority', + origin: string, +): RssItem { + const other = counterparty === 'bidder' ? item.bidderDisplayName : item.authorityName; + const value = item.valueEur != null ? money(item.valueEur) : 'без обявена стойност'; + const parts = [ + `${counterparty === 'bidder' ? 'Изпълнител' : 'Възложител'}: ${other}`, + `Стойност: ${value}`, + item.signedAt ? `Подписан: ${date(item.signedAt)}` : null, + `Процедура: ${item.procedureLabel}`, + ].filter((p): p is string => p !== null); + return { + title: `${item.subject} - ${other}`, + link: `${origin}/contracts/${item.id}`, + description: parts.join(' · '), + pubDate: rssDate(item.signedAt), + }; +} + +export function rssFeed(opts: { + title: string; + description: string; + /** Absolute URL of the HTML profile the feed mirrors. */ + siteLink: string; + /** Absolute URL of the feed itself (atom:link rel="self"). */ + selfLink: string; + items: RssItem[]; +}): string { + const items = opts.items + .map((item) => + [ + ' ', + ` ${xmlEscape(item.title)}`, + ` ${xmlEscape(item.link)}`, + ` ${xmlEscape(item.link)}`, + item.pubDate ? ` ${xmlEscape(item.pubDate)}` : null, + ` ${xmlEscape(item.description)}`, + ' ', + ] + .filter((line): line is string => line !== null) + .join('\n'), + ) + .join('\n'); + // Channel-level pubDate comes from the newest item so the output is a pure function of the data + // (deterministic for tests and for the edge cache) - no "now" timestamp anywhere. + const newest = opts.items.find((item) => item.pubDate != null)?.pubDate; + return [ + '', + '', + ' ', + ` ${xmlEscape(opts.title)}`, + ` ${xmlEscape(opts.siteLink)}`, + ` ${xmlEscape(opts.description)}`, + ' bg', + newest ? ` ${xmlEscape(newest)}` : null, + ` `, + items || null, + ' ', + '', + '', + ] + .filter((line): line is string => line !== null) + .join('\n'); +} diff --git a/apps/web/app/routes.ts b/apps/web/app/routes.ts index 70909b7d..f8dad684 100644 --- a/apps/web/app/routes.ts +++ b/apps/web/app/routes.ts @@ -13,9 +13,11 @@ export default [ route('analytics', 'routes/analytics.tsx'), route('companies', 'routes/companies.tsx'), route('companies.csv', 'routes/companies.csv.tsx'), + route('companies/:eik.rss', 'routes/company.rss.tsx'), route('companies/:eik', 'routes/company.tsx'), route('authorities', 'routes/authorities.tsx'), route('authorities.csv', 'routes/authorities.csv.tsx'), + route('authorities/:eik.rss', 'routes/authority.rss.tsx'), route('authorities/:eik', 'routes/authority.tsx'), route('contracts', 'routes/contracts.tsx'), route('contracts.csv', 'routes/contracts.csv.tsx'), diff --git a/apps/web/app/routes/authority.rss.tsx b/apps/web/app/routes/authority.rss.tsx new file mode 100644 index 00000000..099fb55d --- /dev/null +++ b/apps/web/app/routes/authority.rss.tsx @@ -0,0 +1,38 @@ +import { authorityIdFromSlug, getAuthorityHead, listRecentEntityContracts } from '@sigma/db'; +import type { Route } from './+types/authority.rss'; +import { publicCache } from '../lib/cache'; +import { withDataSource } from '../lib/dataSource'; +import { contractRssItem, rssFeed } from '../lib/feed'; +import { withDbRetry } from '../lib/retry'; + +// Resource route: RSS 2.0 feed of an authority's newest contracts (/authorities/:eik.rss) - the +// no-account way to follow an entity (docs/api.md). X-Robots-Tag keeps feeds out of search indexes +// (profile pages carry the indexable content; some company profiles are deliberately noindex, #173). +export async function loader({ params, request, context }: Route.LoaderArgs) { + const eik = (params.eik ?? '').replace(/\.rss$/, ''); + if (!eik.trim()) return withDataSource(new Response('Not Found', { status: 404 })); + const db = context.cloudflare.env.DB; + const authorityId = authorityIdFromSlug(eik); + const { origin } = new URL(request.url); + return withDbRetry(async () => { + const head = await getAuthorityHead(db, authorityId); + if (!head) return withDataSource(new Response('Not Found', { status: 404 })); + const contracts = await listRecentEntityContracts(db, { kind: 'authority', authorityId }); + const xml = rssFeed({ + title: `${head.name} - нови договори - СИГМА`, + description: `Най-новите договори за обществени поръчки, възложени от ${head.name}.`, + siteLink: `${origin}/authorities/${eik}`, + selfLink: `${origin}/authorities/${eik}.rss`, + items: contracts.map((c) => contractRssItem(c, 'bidder', origin)), + }); + return withDataSource( + new Response(xml, { + headers: { + 'Content-Type': 'application/rss+xml; charset=utf-8', + 'Cache-Control': publicCache(3600), + 'X-Robots-Tag': 'noindex', + }, + }), + ); + }); +} diff --git a/apps/web/app/routes/authority.tsx b/apps/web/app/routes/authority.tsx index 20298c1b..7485f18d 100644 --- a/apps/web/app/routes/authority.tsx +++ b/apps/web/app/routes/authority.tsx @@ -348,9 +348,17 @@ export default function Authority({ loaderData }: Route.ComponentProps) { Виж всички / филтрирай / свали като CSV → + {' · '} + Следи новите договори (RSS)

+ ); } diff --git a/apps/web/app/routes/company.rss.tsx b/apps/web/app/routes/company.rss.tsx new file mode 100644 index 00000000..dd4a5343 --- /dev/null +++ b/apps/web/app/routes/company.rss.tsx @@ -0,0 +1,39 @@ +import { bidderIdFromSlug, getCompanyHead, listRecentEntityContracts } from '@sigma/db'; +import type { Route } from './+types/company.rss'; +import { publicCache } from '../lib/cache'; +import { withDataSource } from '../lib/dataSource'; +import { contractRssItem, rssFeed } from '../lib/feed'; +import { withDbRetry } from '../lib/retry'; + +// Resource route: RSS 2.0 feed of a company's newest contracts (/companies/:eik.rss) - the +// no-account way to follow an entity (docs/api.md). X-Robots-Tag keeps feeds out of search indexes +// (profile pages carry the indexable content; some company profiles are deliberately noindex, #173). +export async function loader({ params, request, context }: Route.LoaderArgs) { + const slug = (params.eik ?? '').replace(/\.rss$/, ''); + if (!slug.trim()) return withDataSource(new Response('Not Found', { status: 404 })); + const bidderId = bidderIdFromSlug(slug); + if (!bidderId) return withDataSource(new Response('Not Found', { status: 404 })); + const db = context.cloudflare.env.DB; + const { origin } = new URL(request.url); + return withDbRetry(async () => { + const head = await getCompanyHead(db, bidderId); + if (!head) return withDataSource(new Response('Not Found', { status: 404 })); + const contracts = await listRecentEntityContracts(db, { kind: 'company', bidderId }); + const xml = rssFeed({ + title: `${head.name} - нови договори - СИГМА`, + description: `Най-новите договори за обществени поръчки, спечелени от ${head.name}.`, + siteLink: `${origin}/companies/${slug}`, + selfLink: `${origin}/companies/${slug}.rss`, + items: contracts.map((c) => contractRssItem(c, 'authority', origin)), + }); + return withDataSource( + new Response(xml, { + headers: { + 'Content-Type': 'application/rss+xml; charset=utf-8', + 'Cache-Control': publicCache(3600), + 'X-Robots-Tag': 'noindex', + }, + }), + ); + }); +} diff --git a/apps/web/app/routes/company.tsx b/apps/web/app/routes/company.tsx index 29caf1c8..4895204c 100644 --- a/apps/web/app/routes/company.tsx +++ b/apps/web/app/routes/company.tsx @@ -408,9 +408,17 @@ export default function Company({ loaderData }: Route.ComponentProps) { Виж всички / филтрирай / свали като CSV → + {' · '} + Следи новите договори (RSS)

+ ); } From 5e225c6d4e13caa065f754853135d31ac2aac635 Mon Sep 17 00:00:00 2001 From: Rumen Slavov <26761822+B353N@users.noreply.github.com> Date: Sat, 4 Jul 2026 11:24:51 +0300 Subject: [PATCH 3/9] docs(api): document the profile rss feeds --- README.md | 2 +- docs/api.md | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1c3deecc..a85124bc 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ | Потоци | `/flows` | Парични потоци възложител → компания (суми + брой) | | Търсене | `/search` | По имена, предмет и идентификатори | -Списъците имат CSV експорт (`/contracts.csv`, `/companies.csv`, `/authorities.csv`), всеки договор — JSON изглед (`/contracts/[id].json`). Има и страници за методология, достъпност, поверителност и impressum, плюс `robots.txt` и sitemap файлове. +Списъците имат CSV експорт (`/contracts.csv`, `/companies.csv`, `/authorities.csv`), всеки договор — JSON изглед (`/contracts/[id].json`), а профилите на институции и компании — RSS фийд с най-новите им договори (`.rss`). Има и страници за методология, достъпност, поверителност и impressum, плюс `robots.txt` и sitemap файлове. ## Накъде върви diff --git a/docs/api.md b/docs/api.md index f4bf0b24..9b18e776 100644 --- a/docs/api.md +++ b/docs/api.md @@ -54,6 +54,18 @@ HTML-страниците, всеки списък и всеки договор Това е единственият per-entity JSON днес; за институции/компании ползвайте CSV списъците или HTML профилите. +### RSS фийдове на профилите + +`GET /authorities/{ЕИК}.rss` и `GET /companies/{slug}.rss` → RSS 2.0 +(`application/rss+xml; charset=utf-8`) с най-новите до 50 договора на +институцията / компанията, подредени по дата на подписване (при липсваща дата - +по публикуване). Всеки запис носи предмета, насрещната страна, стойността и +процедурата; ``/`` водят към страницата на договора. Това е +първата стъпка на „наблюдаваните списъци": следене на субект без акаунт - от +RSS четец или автоматизация. 404 за непознат профил; фийдовете носят +`X-Robots-Tag: noindex` (индексируемото съдържание е HTML профилът, а +разпознатите ЕТ профили са умишлено noindex - вж. бележката за личните данни). + ### Sitemap-и `GET /sitemap.xml` (индекс) + `/sitemap-pages.xml`, `/sitemap-authorities.xml`, @@ -90,4 +102,5 @@ endpoint и **няма** обща REST заявка отвъд изброено Този документ покрива наличното днес. Ако ви трябва формат или endpoint, който липсва (напр. OCDS пакети, per-entity JSON за институции/компании, годишни -bulk dump-ове), отворете issue — посоката е координирана в `docs/`. +bulk dump-ове, email известия върху RSS фийдовете), отворете issue — посоката е +координирана в `docs/`. From c994987447906351ebf4f83458832e6938c3bc0d Mon Sep 17 00:00:00 2001 From: Rumen Slavov <26761822+B353N@users.noreply.github.com> Date: Thu, 9 Jul 2026 09:17:00 +0300 Subject: [PATCH 4/9] fix(web): rss pubDate falls back to publishedAt + loader tests (review) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address the review notes on the profile RSS feeds: 1. pubDate used only signedAt, but the query orders by COALESCE(signed_at, published_at). A contract positioned as "new" by publish date came out with no (readers cannot sort it) and could make the channel older than the true newest item. ContractListItem now carries publishedAt (added to the shared SELECT + toItem), and contractRssItem falls back to it — matching the ordering and the docs' "при липсваща дата - по публикуване" (review ydimitrof). 2. Add loader tests for authority.rss / company.rss: the 200 path (self/site links, noindex header, counterparty side — bidder for authority feeds, authority for company feeds), the 404 branches (absent rollup row, empty eik, undecodable company slug before any DB read), and the defensive .rss-suffix strip. The .rss strip in the loaders is kept for parity with the existing :id.json route (routes/contract.json.tsx) and is now locked by a test, rather than removed. --- apps/web/app/lib/feed.test.ts | 13 +++ apps/web/app/lib/feed.ts | 5 +- apps/web/app/routes/authority.rss.test.ts | 92 +++++++++++++++++++++ apps/web/app/routes/company.rss.test.ts | 99 +++++++++++++++++++++++ packages/api-contract/src/index.ts | 1 + packages/db/src/queries/contracts.test.ts | 1 + packages/db/src/queries/contracts.ts | 4 +- 7 files changed, 213 insertions(+), 2 deletions(-) create mode 100644 apps/web/app/routes/authority.rss.test.ts create mode 100644 apps/web/app/routes/company.rss.test.ts diff --git a/apps/web/app/lib/feed.test.ts b/apps/web/app/lib/feed.test.ts index e958de18..b1477414 100644 --- a/apps/web/app/lib/feed.test.ts +++ b/apps/web/app/lib/feed.test.ts @@ -18,6 +18,7 @@ function item(overrides: Partial = {}): ContractListItem { bidderKind: 'company', procedureLabel: 'Открита процедура', signedAt: '2026-05-15', + publishedAt: null, bidsReceived: 3, valueEur: 12345.67, ...overrides, @@ -68,6 +69,18 @@ describe('contractRssItem', () => { expect(rss.description).not.toContain('Подписан:'); expect(rss.pubDate).toBeNull(); }); + + it('falls back to publishedAt for pubDate when there is no signing date', () => { + // Mirrors the query's ORDER BY COALESCE(signed_at, published_at): an item ordered by publish date + // must still carry a so readers can sort it. The description omits „Подписан:" (no signing). + const rss = contractRssItem( + item({ signedAt: null, publishedAt: '2026-05-10' }), + 'bidder', + 'https://x.bg', + ); + expect(rss.pubDate).toBe('Sun, 10 May 2026 00:00:00 GMT'); + expect(rss.description).not.toContain('Подписан:'); + }); }); describe('rssFeed', () => { diff --git a/apps/web/app/lib/feed.ts b/apps/web/app/lib/feed.ts index fbe26a7d..29f3683d 100644 --- a/apps/web/app/lib/feed.ts +++ b/apps/web/app/lib/feed.ts @@ -53,7 +53,10 @@ export function contractRssItem( title: `${item.subject} - ${other}`, link: `${origin}/contracts/${item.id}`, description: parts.join(' · '), - pubDate: rssDate(item.signedAt), + // Fall back to publishedAt when there is no signing date, matching the query's + // `ORDER BY COALESCE(signed_at, published_at)`: an item positioned as "new" by publish date must + // carry a so readers can order it, and the channel pubDate stays the true newest (review ydimitrof). + pubDate: rssDate(item.signedAt ?? item.publishedAt), }; } diff --git a/apps/web/app/routes/authority.rss.test.ts b/apps/web/app/routes/authority.rss.test.ts new file mode 100644 index 00000000..ff325ce4 --- /dev/null +++ b/apps/web/app/routes/authority.rss.test.ts @@ -0,0 +1,92 @@ +import { describe, expect, it } from 'vitest'; +import { loader } from './authority.rss'; + +const contractRow = { + id: 'c:e:UNP-9:1:eik:222222222', + subject: 'Ремонт на път', + unp: 'UNP-9', + cpv_code: '45000000', + eu_funded: 0, + authority_id: 'auth:123456789', + authority_name: 'Община Пример', + bidder_id: 'eik:222222222', + bidder_name: 'Пътстрой ЕООД', + bidder_kind: 'company', + procedure_type: 'Открита процедура', + signed_at: '2026-05-15', + published_at: '2026-05-10', + bids_received: 2, + amount_eur: 1000, +}; + +function fakeDb(head: { name: string } | null, rows: unknown[] = []): D1Database { + return { + prepare(sql: string) { + const stmt = { + bind() { + return stmt; + }, + async first() { + return (sql.includes('authority_totals') ? head : null) as T; + }, + async all() { + return { results: rows as T[] }; + }, + }; + return stmt; + }, + } as unknown as D1Database; +} + +function call(url: string, eik: string, db: D1Database) { + return loader({ + request: new Request(url), + params: { eik }, + context: { cloudflare: { env: { DB: db } } }, + } as unknown as Parameters[0]); +} + +describe('authority.rss loader', () => { + it('serves an RSS feed with self/site links and bidder-side items', async () => { + const res = await call( + 'https://sigma.midt.bg/authorities/123456789.rss', + '123456789', + fakeDb({ name: 'Община Пример' }, [contractRow]), + ); + expect(res.status).toBe(200); + expect(res.headers.get('Content-Type')).toContain('application/rss+xml'); + expect(res.headers.get('X-Robots-Tag')).toBe('noindex'); + + const body = await res.text(); + expect(body).toContain('Община Пример - нови договори - СИГМА'); + expect(body).toContain('https://sigma.midt.bg/authorities/123456789'); + expect(body).toContain( + '', + ); + // An authority feed lists the WINNER (counterparty = 'bidder'). + expect(body).toContain('Изпълнител: Пътстрой ЕООД'); + // contractSlug strips the leading 'c:' from the id. + expect(body).toContain('https://sigma.midt.bg/contracts/e:UNP-9:1:eik:222222222'); + }); + + it('404s for an authority absent from the rollup', async () => { + const res = await call('https://sigma.midt.bg/authorities/999.rss', '999', fakeDb(null)); + expect(res.status).toBe(404); + }); + + it('404s for an empty eik', async () => { + const res = await call('https://sigma.midt.bg/authorities/.rss', '.rss', fakeDb({ name: 'x' })); + expect(res.status).toBe(404); + }); + + it('strips a .rss suffix left in the param so the links are not doubled', async () => { + const res = await call( + 'https://sigma.midt.bg/authorities/123456789.rss', + '123456789.rss', + fakeDb({ name: 'Община Пример' }), + ); + const body = await res.text(); + expect(body).toContain('https://sigma.midt.bg/authorities/123456789'); + expect(body).not.toContain('123456789.rss.rss'); + }); +}); diff --git a/apps/web/app/routes/company.rss.test.ts b/apps/web/app/routes/company.rss.test.ts new file mode 100644 index 00000000..6f7ba119 --- /dev/null +++ b/apps/web/app/routes/company.rss.test.ts @@ -0,0 +1,99 @@ +import { describe, expect, it } from 'vitest'; +import { loader } from './company.rss'; + +const contractRow = { + id: 'c:e:UNP-9:1:eik:222222222', + subject: 'Ремонт на път', + unp: 'UNP-9', + cpv_code: '45000000', + eu_funded: 0, + authority_id: 'auth:123456789', + authority_name: 'Община Пример', + bidder_id: 'eik:222222222', + bidder_name: 'Пътстрой ЕООД', + bidder_kind: 'company', + procedure_type: 'Открита процедура', + signed_at: '2026-05-15', + published_at: '2026-05-10', + bids_received: 2, + amount_eur: 1000, +}; + +function fakeDb(head: { name: string; kind: string } | null, rows: unknown[] = []): D1Database { + return { + prepare(sql: string) { + const stmt = { + bind() { + return stmt; + }, + async first() { + return (sql.includes('company_totals') ? head : null) as T; + }, + async all() { + return { results: rows as T[] }; + }, + }; + return stmt; + }, + } as unknown as D1Database; +} + +function call(url: string, eik: string, db: D1Database) { + return loader({ + request: new Request(url), + params: { eik }, + context: { cloudflare: { env: { DB: db } } }, + } as unknown as Parameters[0]); +} + +describe('company.rss loader', () => { + it('serves an RSS feed with self/site links and authority-side items', async () => { + const res = await call( + 'https://sigma.midt.bg/companies/222222222.rss', + '222222222', + fakeDb({ name: 'Пътстрой ЕООД', kind: 'company' }, [contractRow]), + ); + expect(res.status).toBe(200); + expect(res.headers.get('Content-Type')).toContain('application/rss+xml'); + expect(res.headers.get('X-Robots-Tag')).toBe('noindex'); + + const body = await res.text(); + expect(body).toContain('Пътстрой ЕООД - нови договори - СИГМА'); + expect(body).toContain('https://sigma.midt.bg/companies/222222222'); + expect(body).toContain( + '', + ); + // A company feed lists the BUYER (counterparty = 'authority'). + expect(body).toContain('Възложител: Община Пример'); + }); + + it('404s for a company absent from the rollup', async () => { + const res = await call( + 'https://sigma.midt.bg/companies/222222222.rss', + '222222222', + fakeDb(null), + ); + expect(res.status).toBe(404); + }); + + it('404s for an undecodable slug before touching the DB (no bidder id)', async () => { + // 'xyz' is neither a valid ЕИК nor an `n`-prefixed base64 name slug → bidderIdFromSlug returns null. + const res = await call( + 'https://sigma.midt.bg/companies/xyz.rss', + 'xyz', + fakeDb({ name: 'should not be read', kind: 'company' }), + ); + expect(res.status).toBe(404); + }); + + it('strips a .rss suffix left in the param so the links are not doubled', async () => { + const res = await call( + 'https://sigma.midt.bg/companies/222222222.rss', + '222222222.rss', + fakeDb({ name: 'Пътстрой ЕООД', kind: 'company' }), + ); + const body = await res.text(); + expect(body).toContain('https://sigma.midt.bg/companies/222222222'); + expect(body).not.toContain('222222222.rss.rss'); + }); +}); diff --git a/packages/api-contract/src/index.ts b/packages/api-contract/src/index.ts index 2adfeb29..1fff3e76 100644 --- a/packages/api-contract/src/index.ts +++ b/packages/api-contract/src/index.ts @@ -227,6 +227,7 @@ export interface ContractListItem { bidderKind: EntityKind; procedureLabel: string; signedAt: string | null; + publishedAt: string | null; // ordering/pubDate fallback when signedAt is absent (COALESCE order) bidsReceived: number | null; valueEur: number | null; // null = suspect / unconvertible → render the проверяват note } diff --git a/packages/db/src/queries/contracts.test.ts b/packages/db/src/queries/contracts.test.ts index 810dc079..a9273c73 100644 --- a/packages/db/src/queries/contracts.test.ts +++ b/packages/db/src/queries/contracts.test.ts @@ -33,6 +33,7 @@ const contractRow = { bidder_kind: 'company' as const, procedure_type: 'Открита процедура', signed_at: '2024-01-01', + published_at: '2024-01-02', bids_received: 3, amount_eur: 1000, sort_value: 1000, diff --git a/packages/db/src/queries/contracts.ts b/packages/db/src/queries/contracts.ts index 10b0b825..edffa50b 100644 --- a/packages/db/src/queries/contracts.ts +++ b/packages/db/src/queries/contracts.ts @@ -88,6 +88,7 @@ interface ContractRow { bidder_kind: 'company' | 'consortium'; procedure_type: string; signed_at: string | null; + published_at: string | null; bids_received: number | null; amount_eur: number | null; } @@ -96,7 +97,7 @@ const SELECT = ` SELECT c.id, COALESCE(NULLIF(c.contract_subject, ''), t.title) AS subject, t.source_id AS unp, t.cpv_code, c.eu_funded, t.authority_id, a.name AS authority_name, c.bidder_id, b.name AS bidder_name, b.kind AS bidder_kind, - t.procedure_type, c.signed_at, c.bids_received, c.amount_eur`; + t.procedure_type, c.signed_at, c.published_at, c.bids_received, c.amount_eur`; const FROM = ` FROM contracts c JOIN tenders t ON t.id = c.tender_id @@ -208,6 +209,7 @@ function toItem(r: ContractRow): ContractListItem { bidderKind: r.bidder_kind, procedureLabel: procedureGroup(r.procedure_type).label, signedAt: r.signed_at, + publishedAt: r.published_at, bidsReceived: r.bids_received, valueEur: r.amount_eur, }; From cc0b1928cc9c3e85aa1f0882e3b2bbe0c1bb6d30 Mon Sep 17 00:00:00 2001 From: Rumen Slavov <26761822+B353N@users.noreply.github.com> Date: Thu, 9 Jul 2026 09:56:26 +0300 Subject: [PATCH 5/9] docs(web): explain the intentional authority.rss / company.rss validation asymmetry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit company.rss short-circuits with a 404 when bidderIdFromSlug returns null (name-keyed company slugs need a fallible base64 decode). authority.rss has no equivalent early check because authorityIdFromSlug is total — authorities are always ЕИК-keyed and the raw ЕИК is not format-constrained in the pipeline, so a format guard here could reject a real profile and would diverge from the HTML authority route. Document that, and add a test proving a garbage slug still 404s gracefully via the no-row lookup (review ydimitrof). --- apps/web/app/routes/authority.rss.test.ts | 11 +++++++++++ apps/web/app/routes/authority.rss.tsx | 5 +++++ 2 files changed, 16 insertions(+) diff --git a/apps/web/app/routes/authority.rss.test.ts b/apps/web/app/routes/authority.rss.test.ts index ff325ce4..89a1d655 100644 --- a/apps/web/app/routes/authority.rss.test.ts +++ b/apps/web/app/routes/authority.rss.test.ts @@ -74,6 +74,17 @@ describe('authority.rss loader', () => { expect(res.status).toBe(404); }); + it('404s gracefully for a garbage slug (no early format check, just a no-row lookup)', async () => { + // authorityIdFromSlug is total, so an invalid slug is not rejected up front — it resolves to an + // authority id that simply matches no row → getAuthorityHead null → 404 (review ydimitrof). + const res = await call( + 'https://sigma.midt.bg/authorities/not-an-eik.rss', + 'not-an-eik', + fakeDb(null), + ); + expect(res.status).toBe(404); + }); + it('404s for an empty eik', async () => { const res = await call('https://sigma.midt.bg/authorities/.rss', '.rss', fakeDb({ name: 'x' })); expect(res.status).toBe(404); diff --git a/apps/web/app/routes/authority.rss.tsx b/apps/web/app/routes/authority.rss.tsx index 099fb55d..0cd7ed99 100644 --- a/apps/web/app/routes/authority.rss.tsx +++ b/apps/web/app/routes/authority.rss.tsx @@ -12,6 +12,11 @@ export async function loader({ params, request, context }: Route.LoaderArgs) { const eik = (params.eik ?? '').replace(/\.rss$/, ''); if (!eik.trim()) return withDataSource(new Response('Not Found', { status: 404 })); const db = context.cloudflare.env.DB; + // No early slug-validity 404 like company.rss (which short-circuits on bidderIdFromSlug === null): + // authorityIdFromSlug is total (authorities are always ЕИК-keyed, no fallible decode), and the raw + // ЕИК is not format-constrained in the pipeline, so validating here could reject a real profile and + // would diverge from the HTML authority route. An unknown slug just yields getAuthorityHead === null + // → 404 below, at the cost of one indexed no-row lookup (review ydimitrof). const authorityId = authorityIdFromSlug(eik); const { origin } = new URL(request.url); return withDbRetry(async () => { From c6db83db81f166432c75c8dd00d9035e408d487e Mon Sep 17 00:00:00 2001 From: Rumen Slavov <26761822+B353N@users.noreply.github.com> Date: Thu, 9 Jul 2026 10:14:53 +0300 Subject: [PATCH 6/9] fix(web): canonical rss links + order-independent channel pubDate (review) Address the review notes on the profile RSS feeds: 1. c.published_at confirmed correct, and now proven against the real schema. published_at exists on BOTH contracts and tenders, so a wrong alias would not error - it would silently order by the tender's publish date. Added a real-sqlite test (recent-contracts-schema.test.ts) that seeds a contract whose published_at differs from its tender's and asserts the query reads the CONTRACT column via COALESCE(c.signed_at, c.published_at) - a gap the mock-D1 unit tests cannot cover (review ydimitrof). 2. authority.rss / company.rss now build siteLink/selfLink from the CANONICAL slug (re-derived from the resolved id) instead of the raw request param, so a resolvable-but-non-canonical request (e.g. a name-keyed company slug with different base64 padding) emits the same URLs as the HTML profile. Test proves 'nWA==' canonicalises to 'nWA' in the links. 3. rssFeed computes the channel pubDate as the MAX over all items instead of find-first, so it no longer silently depends on the caller passing items newest-first. Test feeds mis-ordered items and asserts the channel pubDate is the newest. --- apps/web/app/lib/feed.test.ts | 12 ++++ apps/web/app/lib/feed.ts | 11 +++- apps/web/app/routes/authority.rss.tsx | 15 ++++- apps/web/app/routes/company.rss.test.ts | 16 +++++ apps/web/app/routes/company.rss.tsx | 15 ++++- .../db/src/recent-contracts-schema.test.ts | 63 +++++++++++++++++++ 6 files changed, 123 insertions(+), 9 deletions(-) create mode 100644 packages/db/src/recent-contracts-schema.test.ts diff --git a/apps/web/app/lib/feed.test.ts b/apps/web/app/lib/feed.test.ts index b1477414..1732fe32 100644 --- a/apps/web/app/lib/feed.test.ts +++ b/apps/web/app/lib/feed.test.ts @@ -135,4 +135,16 @@ describe('rssFeed', () => { expect(xml).not.toContain(''); expect(xml).not.toContain(''); }); + + it('channel pubDate is the MAX item date, not the first — even if items are mis-ordered', () => { + // rssFeed cannot enforce the caller's newest-first order, so it computes the max defensively. + const older = contractRssItem(item({ signedAt: '2026-01-10' }), 'bidder', 'https://x.bg'); + const newer = contractRssItem(item({ signedAt: '2026-08-20' }), 'bidder', 'https://x.bg'); + // Newest is SECOND in the array → a `find`-first would wrongly pick the older one. Anchor on the + // channel's line (only the channel pubDate follows it) to target the channel, not items. + const xml = rssFeed({ ...opts, items: [older, newer] }); + expect(xml).toContain( + 'bg\n Thu, 20 Aug 2026 00:00:00 GMT', + ); + }); }); diff --git a/apps/web/app/lib/feed.ts b/apps/web/app/lib/feed.ts index 29f3683d..554b2ae3 100644 --- a/apps/web/app/lib/feed.ts +++ b/apps/web/app/lib/feed.ts @@ -84,9 +84,14 @@ export function rssFeed(opts: { .join('\n'), ) .join('\n'); - // Channel-level pubDate comes from the newest item so the output is a pure function of the data - // (deterministic for tests and for the edge cache) - no "now" timestamp anywhere. - const newest = opts.items.find((item) => item.pubDate != null)?.pubDate; + // Channel-level pubDate is the newest item's date, computed as the MAX over all items rather than + // trusting input order: rssFeed is generic and cannot enforce the caller's newest-first ordering, so + // a future reordering must not silently yield a wrong channel (review ydimitrof). Still a + // pure function of the data — no "now" timestamp anywhere (RFC-822 dates compare via Date.parse). + const newest = opts.items.reduce((max, item) => { + if (!item.pubDate) return max; + return max === undefined || Date.parse(item.pubDate) > Date.parse(max) ? item.pubDate : max; + }, undefined); return [ '', '', diff --git a/apps/web/app/routes/authority.rss.tsx b/apps/web/app/routes/authority.rss.tsx index 0cd7ed99..743b0db8 100644 --- a/apps/web/app/routes/authority.rss.tsx +++ b/apps/web/app/routes/authority.rss.tsx @@ -1,4 +1,9 @@ -import { authorityIdFromSlug, getAuthorityHead, listRecentEntityContracts } from '@sigma/db'; +import { + authorityIdFromSlug, + authoritySlug, + getAuthorityHead, + listRecentEntityContracts, +} from '@sigma/db'; import type { Route } from './+types/authority.rss'; import { publicCache } from '../lib/cache'; import { withDataSource } from '../lib/dataSource'; @@ -18,6 +23,10 @@ export async function loader({ params, request, context }: Route.LoaderArgs) { // would diverge from the HTML authority route. An unknown slug just yields getAuthorityHead === null // → 404 below, at the cost of one indexed no-row lookup (review ydimitrof). const authorityId = authorityIdFromSlug(eik); + // Build the self/site links from the CANONICAL slug (derived from the resolved id), not the raw + // request param, so a resolvable-but-non-canonical request still emits the same URLs as the HTML + // profile (review ydimitrof). + const canonicalEik = authoritySlug(authorityId); const { origin } = new URL(request.url); return withDbRetry(async () => { const head = await getAuthorityHead(db, authorityId); @@ -26,8 +35,8 @@ export async function loader({ params, request, context }: Route.LoaderArgs) { const xml = rssFeed({ title: `${head.name} - нови договори - СИГМА`, description: `Най-новите договори за обществени поръчки, възложени от ${head.name}.`, - siteLink: `${origin}/authorities/${eik}`, - selfLink: `${origin}/authorities/${eik}.rss`, + siteLink: `${origin}/authorities/${canonicalEik}`, + selfLink: `${origin}/authorities/${canonicalEik}.rss`, items: contracts.map((c) => contractRssItem(c, 'bidder', origin)), }); return withDataSource( diff --git a/apps/web/app/routes/company.rss.test.ts b/apps/web/app/routes/company.rss.test.ts index 6f7ba119..dc7a7d02 100644 --- a/apps/web/app/routes/company.rss.test.ts +++ b/apps/web/app/routes/company.rss.test.ts @@ -96,4 +96,20 @@ describe('company.rss loader', () => { expect(body).toContain('https://sigma.midt.bg/companies/222222222'); expect(body).not.toContain('222222222.rss.rss'); }); + + it('builds self/site links from the canonical slug, not the raw request param', async () => { + // 'nWA==' is a non-canonical encoding of the name-keyed slug 'nWA' (companySlug strips the '=' + // base64 padding). The links must use the canonical 'nWA', matching the HTML profile. + const res = await call( + 'https://sigma.midt.bg/companies/nWA==.rss', + 'nWA==', + fakeDb({ name: 'X', kind: 'company' }), + ); + const body = await res.text(); + expect(body).toContain('https://sigma.midt.bg/companies/nWA'); + expect(body).toContain( + '', + ); + expect(body).not.toContain('nWA=='); + }); }); diff --git a/apps/web/app/routes/company.rss.tsx b/apps/web/app/routes/company.rss.tsx index dd4a5343..4373f412 100644 --- a/apps/web/app/routes/company.rss.tsx +++ b/apps/web/app/routes/company.rss.tsx @@ -1,4 +1,9 @@ -import { bidderIdFromSlug, getCompanyHead, listRecentEntityContracts } from '@sigma/db'; +import { + bidderIdFromSlug, + companySlug, + getCompanyHead, + listRecentEntityContracts, +} from '@sigma/db'; import type { Route } from './+types/company.rss'; import { publicCache } from '../lib/cache'; import { withDataSource } from '../lib/dataSource'; @@ -13,6 +18,10 @@ export async function loader({ params, request, context }: Route.LoaderArgs) { if (!slug.trim()) return withDataSource(new Response('Not Found', { status: 404 })); const bidderId = bidderIdFromSlug(slug); if (!bidderId) return withDataSource(new Response('Not Found', { status: 404 })); + // Build the self/site links from the CANONICAL slug (re-derived from the resolved bidder id), not the + // raw request param, so a resolvable-but-non-canonical request (e.g. a name-keyed slug with different + // base64 padding) still emits the same URLs as the HTML profile (review ydimitrof). + const canonicalSlug = companySlug(bidderId); const db = context.cloudflare.env.DB; const { origin } = new URL(request.url); return withDbRetry(async () => { @@ -22,8 +31,8 @@ export async function loader({ params, request, context }: Route.LoaderArgs) { const xml = rssFeed({ title: `${head.name} - нови договори - СИГМА`, description: `Най-новите договори за обществени поръчки, спечелени от ${head.name}.`, - siteLink: `${origin}/companies/${slug}`, - selfLink: `${origin}/companies/${slug}.rss`, + siteLink: `${origin}/companies/${canonicalSlug}`, + selfLink: `${origin}/companies/${canonicalSlug}.rss`, items: contracts.map((c) => contractRssItem(c, 'authority', origin)), }); return withDataSource( diff --git a/packages/db/src/recent-contracts-schema.test.ts b/packages/db/src/recent-contracts-schema.test.ts new file mode 100644 index 00000000..ba93764a --- /dev/null +++ b/packages/db/src/recent-contracts-schema.test.ts @@ -0,0 +1,63 @@ +/// +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +// listRecentEntityContracts (queries/contracts.ts) selects `c.published_at` and orders by +// COALESCE(c.signed_at, c.published_at). `published_at` exists on BOTH `contracts` and `tenders`, so a +// wrong alias (`t.published_at`) would NOT error — it would silently order by the tender's publish date +// instead of the contract's, and the mock-D1 unit tests can't catch that. This runs the real SELECT +// shape against the real migrated schema, with the contract's and tender's published_at deliberately +// different, to prove the query reads the CONTRACT column (review ydimitrof). + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); +const migration0 = resolve(root, 'packages/db/migrations/0000_init.sql'); + +function sqliteJson(dbPath: string, sql: string): T[] { + const out = execFileSync('sqlite3', ['-json', dbPath, sql], { encoding: 'utf8' }).trim(); + return out ? (JSON.parse(out) as T[]) : []; +} + +describe('listRecentEntityContracts column/alias against the real schema', () => { + let dir: string; + let dbPath: string; + + beforeAll(() => { + dir = mkdtempSync(resolve(tmpdir(), 'sigma-recent-')); + dbPath = resolve(dir, 'test.sqlite'); + execFileSync('sqlite3', ['-bail', dbPath], { input: `.read ${migration0}\n`, stdio: 'pipe' }); + execFileSync('sqlite3', ['-bail', dbPath], { + input: [ + `INSERT INTO authorities(id,name) VALUES('auth:1','A');`, + `INSERT INTO bidders(id,name) VALUES('eik:1','B');`, + // Tender publish date is DELIBERATELY different from the contract's. + `INSERT INTO tenders(id,source_id,title,authority_id,cpv_code,procedure_type,status,published_at) ` + + `VALUES('t:1','U1','T','auth:1','45000000','открита','awarded','2020-01-01');`, + // No signing date → the ORDER BY falls back to the contract's published_at. + `INSERT INTO contracts(id,tender_id,bidder_id,amount,amount_eur,currency,value_flag,signed_at,published_at,bids_received) ` + + `VALUES('c:1','t:1','eik:1',1000,1000,'EUR','ok',NULL,'2024-09-09',1);`, + ].join('\n'), + stdio: 'pipe', + }); + }); + + afterAll(() => rmSync(dir, { recursive: true, force: true })); + + it('reads c.published_at (the contract column), distinct from t.published_at', () => { + const rows = sqliteJson<{ cpub: string; tpub: string; ord: string }>( + dbPath, + `SELECT c.published_at AS cpub, t.published_at AS tpub, + COALESCE(c.signed_at, c.published_at) AS ord + FROM contracts c JOIN tenders t ON t.id = c.tender_id + WHERE t.authority_id = 'auth:1'`, + ); + expect(rows).toHaveLength(1); + expect(rows[0]?.cpub).toBe('2024-09-09'); // contract's own publish date + expect(rows[0]?.tpub).toBe('2020-01-01'); // the tender's — proves the two columns differ + // The recency fallback used by listRecentEntityContracts must resolve to the CONTRACT's date. + expect(rows[0]?.ord).toBe('2024-09-09'); + }); +}); From c132f8b92011f344884b9b5cc561fbc00b3c925c Mon Sep 17 00:00:00 2001 From: Rumen Slavov <26761822+B353N@users.noreply.github.com> Date: Thu, 9 Jul 2026 10:29:58 +0300 Subject: [PATCH 7/9] fix(web): strip XML-1.0-invalid control chars in the rss feed; clarify pubDate ?? MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - xmlEscape now removes control characters that are illegal in XML 1.0 even when entity-escaped (everything below U+0020 except TAB/LF/CR) before escaping the five entities. A stray one in a source subject/name would otherwise make the whole feed invalid XML and get it rejected by strict readers (review ydimitrof). Test proves the invalid controls are dropped and tab/newline/CR are preserved. - Document that `signedAt ?? publishedAt` deliberately mirrors SQL COALESCE (empty string is treated as present; only NULL falls through), so pubDate stays in sync with the row's ordering position. No behaviour change — real data has NULL signed_at. --- apps/web/app/lib/feed.test.ts | 7 +++++++ apps/web/app/lib/feed.ts | 13 +++++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/apps/web/app/lib/feed.test.ts b/apps/web/app/lib/feed.test.ts index 1732fe32..8c2454de 100644 --- a/apps/web/app/lib/feed.test.ts +++ b/apps/web/app/lib/feed.test.ts @@ -32,6 +32,13 @@ describe('xmlEscape', () => { it('passes ordinary Bulgarian text through unchanged', () => { expect(xmlEscape('Община „Пример“ - договори')).toBe('Община „Пример“ - договори'); }); + it('drops XML-1.0-invalid control characters but keeps tab / newline / CR', () => { + const c = (n: number) => String.fromCharCode(n); + // U+0000, U+0008, U+001F are illegal in XML 1.0 → stripped; the surrounding text survives. + expect(xmlEscape(`a${c(0)}b${c(8)}c${c(0x1f)}d`)).toBe('abcd'); + // TAB (U+0009), LF (U+000A), CR (U+000D) are legal → preserved verbatim. + expect(xmlEscape(`x${c(9)}y${c(10)}z${c(13)}w`)).toBe(`x${c(9)}y${c(10)}z${c(13)}w`); + }); }); describe('rssDate', () => { diff --git a/apps/web/app/lib/feed.ts b/apps/web/app/lib/feed.ts index 554b2ae3..a9af6fba 100644 --- a/apps/web/app/lib/feed.ts +++ b/apps/web/app/lib/feed.ts @@ -13,8 +13,14 @@ const XML_ESCAPES: Record = { "'": ''', }; +// Control characters that are NOT legal in XML 1.0 even when entity-escaped: everything below U+0020 +// except TAB (U+0009), LF (U+000A) and CR (U+000D). A stray one in a source subject/name would make +// the whole feed invalid XML and strict readers reject it, so drop them before escaping (review +// ydimitrof). There is no meaningful replacement — they carry no display value. +const XML_INVALID_CONTROL = /[\u0000-\u0008\u000B\u000C\u000E-\u001F]/g; + export function xmlEscape(value: string): string { - return value.replace(/[&<>"']/g, (ch) => XML_ESCAPES[ch] ?? ch); + return value.replace(XML_INVALID_CONTROL, '').replace(/[&<>"']/g, (ch) => XML_ESCAPES[ch] ?? ch); } /** 'YYYY-MM-DD' -> RFC 822 (RSS pubDate); null for absent or malformed dates. */ @@ -55,7 +61,10 @@ export function contractRssItem( description: parts.join(' · '), // Fall back to publishedAt when there is no signing date, matching the query's // `ORDER BY COALESCE(signed_at, published_at)`: an item positioned as "new" by publish date must - // carry a so readers can order it, and the channel pubDate stays the true newest (review ydimitrof). + // carry a so readers can order it, and the channel pubDate stays the true newest (review + // ydimitrof). `??` (not `||`) is deliberate: it mirrors SQL COALESCE, which treats an empty string + // as PRESENT (only NULL falls through). Real data has NULL signed_at, so the two never disagree — + // and keeping the same rule here holds pubDate in sync with the row's ordering position. pubDate: rssDate(item.signedAt ?? item.publishedAt), }; } From eab4dd7d8d519d3a9c6de1f89fc75754fbf2914d Mon Sep 17 00:00:00 2001 From: Rumen Slavov <26761822+B353N@users.noreply.github.com> Date: Thu, 9 Jul 2026 10:43:25 +0300 Subject: [PATCH 8/9] style(web): spell XML_INVALID_CONTROL with \xNN hex escapes (review readability) The committed \uNNNN form was byte-exact correct (backslash-u escape sequences, verified by hexdump; the regex matches exactly U+0000-0008, 0B, 0C, 0E-1F and no printable ASCII - probed codepoint-by-codepoint in Node, and the named tests pass). The review read it as caret notation / a literal @-^ range, so switch to the suggested \xNN spelling - proven identical over U+0000..U+2FFF - to remove the ambiguity for human readers. No behaviour change. --- apps/web/app/lib/feed.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/web/app/lib/feed.ts b/apps/web/app/lib/feed.ts index a9af6fba..ff86fad4 100644 --- a/apps/web/app/lib/feed.ts +++ b/apps/web/app/lib/feed.ts @@ -17,7 +17,7 @@ const XML_ESCAPES: Record = { // except TAB (U+0009), LF (U+000A) and CR (U+000D). A stray one in a source subject/name would make // the whole feed invalid XML and strict readers reject it, so drop them before escaping (review // ydimitrof). There is no meaningful replacement — they carry no display value. -const XML_INVALID_CONTROL = /[\u0000-\u0008\u000B\u000C\u000E-\u001F]/g; +const XML_INVALID_CONTROL = /[\x00-\x08\x0B\x0C\x0E-\x1F]/g; export function xmlEscape(value: string): string { return value.replace(XML_INVALID_CONTROL, '').replace(/[&<>"']/g, (ch) => XML_ESCAPES[ch] ?? ch); From 17fcacb318a8be5b93f3d115be569ba41eca4b64 Mon Sep 17 00:00:00 2001 From: Rumen Slavov <26761822+B353N@users.noreply.github.com> Date: Fri, 10 Jul 2026 16:49:09 +0300 Subject: [PATCH 9/9] perf(db): scoped indexes for the entity RSS feeds (fix DoW on the sort) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit listRecentEntityContracts sorts an entity's contracts by COALESCE(signed_at, published_at) DESC with no matching index, so a big supplier/ministry feed gathered ALL the entity's rows and did USE TEMP B-TREE FOR ORDER BY to return 50 — a public, unauthenticated Denial-of-Wallet (D1 bills rows SCANNED), review lyubomir-bozhinov. Migration 0006: - idx_contracts_bidder_recent(bidder_id, COALESCE(signed_at, published_at) DESC, id DESC) — the company feed now index-walks scoped to the bidder. - authority_id lives on tenders, not contracts, so a scoped index was impossible. Denormalise it onto the contract row (ALTER + backfill), keep it populated in the ETL (normalize-raw.sql full rebuild + refresh-slice.sql touched-window UPDATE), index it, and scope the authority feed on c.authority_id. The feed now index-walks. EXPLAIN QUERY PLAN test proves both feeds temp-B-tree-sort BEFORE and index-walk (no sort step) AFTER, on a real sqlite3. refresh-slice.test.ts now applies the full migration chain so its pipeline run sees the new column. ship-domain/compare-served copy columns dynamically, so they carry authority_id with no change. Migration numbered 0006 to clear the in-flight 0002–0005; final order is the maintainer's call. --- .../migrations/0006_recent_feed_indexes.sql | 23 ++++ packages/db/src/queries/contracts.test.ts | 6 +- packages/db/src/queries/contracts.ts | 5 +- packages/db/src/recent-feed-index.test.ts | 112 ++++++++++++++++++ packages/db/src/refresh-slice.test.ts | 25 ++-- scripts/normalize-raw.sql | 7 ++ scripts/refresh-slice.sql | 8 ++ 7 files changed, 176 insertions(+), 10 deletions(-) create mode 100644 packages/db/migrations/0006_recent_feed_indexes.sql create mode 100644 packages/db/src/recent-feed-index.test.ts diff --git a/packages/db/migrations/0006_recent_feed_indexes.sql b/packages/db/migrations/0006_recent_feed_indexes.sql new file mode 100644 index 00000000..f035bee4 --- /dev/null +++ b/packages/db/migrations/0006_recent_feed_indexes.sql @@ -0,0 +1,23 @@ +-- Entity "newest contracts" feeds (RSS) paginate with +-- ORDER BY COALESCE(signed_at, published_at) DESC, id DESC LIMIT 50 +-- scoped to one company or one authority. Without a matching index SQLite gathers ALL of the entity's +-- contracts and does a full USE TEMP B-TREE FOR ORDER BY to return 50 — a public, unauthenticated +-- Denial-of-Wallet vector on a big supplier / ministry (D1 bills rows SCANNED). These two composite +-- indexes let each feed walk the index scoped to the entity and stop at LIMIT. + +-- Company feed: WHERE bidder_id = ? (already a column on contracts). +CREATE INDEX IF NOT EXISTS idx_contracts_bidder_recent + ON contracts(bidder_id, COALESCE(signed_at, published_at) DESC, id DESC); + +-- Authority feed: WHERE authority_id = ?. authority_id lives on `tenders`, not `contracts`, so a +-- scoped index is impossible without denormalising it onto the contract row. Add the column, backfill +-- it from the parent tender, and index it. The ETL keeps it populated (scripts/normalize-raw.sql and +-- scripts/refresh-slice.sql set it right after inserting contracts). SQLite ALTER ADD COLUMN has no +-- IF NOT EXISTS, but migrations apply once; the backfill covers rows that predate this migration. +ALTER TABLE contracts ADD COLUMN authority_id TEXT; +UPDATE contracts + SET authority_id = (SELECT t.authority_id FROM tenders t WHERE t.id = contracts.tender_id) + WHERE authority_id IS NULL; + +CREATE INDEX IF NOT EXISTS idx_contracts_authority_recent + ON contracts(authority_id, COALESCE(signed_at, published_at) DESC, id DESC); diff --git a/packages/db/src/queries/contracts.test.ts b/packages/db/src/queries/contracts.test.ts index a9273c73..117f0497 100644 --- a/packages/db/src/queries/contracts.test.ts +++ b/packages/db/src/queries/contracts.test.ts @@ -156,7 +156,7 @@ describe('listRecentEntityContracts', () => { return { db, captured }; } - it('scopes an authority feed by t.authority_id and orders date-desc with id tiebreak', async () => { + it('scopes an authority feed by c.authority_id and orders date-desc with id tiebreak', async () => { const { db, captured } = capturingDb(); const items = await listRecentEntityContracts(db, { kind: 'authority', @@ -164,7 +164,9 @@ describe('listRecentEntityContracts', () => { }); expect(captured).toHaveLength(1); - expect(captured[0]?.sql).toContain('WHERE t.authority_id = ?'); + // Scoped on the denormalised contract column (migration 0006), not t.authority_id, so it can use + // idx_contracts_authority_recent. + expect(captured[0]?.sql).toContain('WHERE c.authority_id = ?'); expect(captured[0]?.sql).toContain( 'ORDER BY COALESCE(c.signed_at, c.published_at) DESC, c.id DESC', ); diff --git a/packages/db/src/queries/contracts.ts b/packages/db/src/queries/contracts.ts index edffa50b..a2bd7b5d 100644 --- a/packages/db/src/queries/contracts.ts +++ b/packages/db/src/queries/contracts.ts @@ -247,7 +247,10 @@ export async function listRecentEntityContracts( entity: { kind: 'authority'; authorityId: string } | { kind: 'company'; bidderId: string }, limit = 50, ): Promise { - const scope = entity.kind === 'authority' ? 't.authority_id = ?' : 'c.bidder_id = ?'; + // Scope on the contract-row columns (bidder_id, and the denormalised authority_id — migration 0006) + // so the ORDER BY walks idx_contracts_{bidder,authority}_recent scoped to the entity and stops at + // LIMIT, instead of gathering all of the entity's contracts and temp-B-tree-sorting them (DoW). + const scope = entity.kind === 'authority' ? 'c.authority_id = ?' : 'c.bidder_id = ?'; const id = entity.kind === 'authority' ? entity.authorityId : entity.bidderId; const rows = await db .prepare( diff --git a/packages/db/src/recent-feed-index.test.ts b/packages/db/src/recent-feed-index.test.ts new file mode 100644 index 00000000..41e84afb --- /dev/null +++ b/packages/db/src/recent-feed-index.test.ts @@ -0,0 +1,112 @@ +/// +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, readdirSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +// The entity "newest contracts" RSS feeds (listRecentEntityContracts) run, scoped to one company or +// authority, `ORDER BY COALESCE(signed_at, published_at) DESC, id DESC LIMIT 50`. Without a matching +// scoped index the planner gathers ALL of the entity's contracts and does USE TEMP B-TREE FOR ORDER +// BY — a public, unauthenticated Denial-of-Wallet on a big supplier/ministry (D1 bills rows scanned). +// Migration 0006 adds the composite indexes (and denormalises authority_id onto contracts so the +// authority scope is a contract-row column). This proves, on a real sqlite3 without ANALYZE (matching +// D1), that BEFORE the migration both feeds temp-B-tree-sort, and AFTER each walks its scoped index. + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); +const migrationsDir = resolve(root, 'packages/db/migrations'); +const feedMigration = readdirSync(migrationsDir).find((f) => f.includes('recent_feed_indexes')); +if (!feedMigration) throw new Error('recent_feed_indexes migration not found'); +const baseMigrations = readdirSync(migrationsDir) + .filter((f) => f.endsWith('.sql') && f !== feedMigration) + .sort(); + +function readScript(dbPath: string, file: string): void { + execFileSync('sqlite3', ['-bail', dbPath], { + input: `.read ${resolve(migrationsDir, file)}\n`, + stdio: 'pipe', + }); +} + +function exec(dbPath: string, sql: string): void { + execFileSync('sqlite3', ['-bail', dbPath], { input: sql, stdio: 'pipe' }); +} + +function plan(dbPath: string, sql: string): string { + return execFileSync('sqlite3', [dbPath], { + input: `EXPLAIN QUERY PLAN ${sql}\n`, + encoding: 'utf8', + }); +} + +const FROM = + 'FROM contracts c JOIN tenders t ON t.id = c.tender_id ' + + 'JOIN authorities a ON a.id = t.authority_id JOIN bidders b ON b.id = c.bidder_id'; +const ORDER = 'ORDER BY COALESCE(c.signed_at, c.published_at) DESC, c.id DESC LIMIT 50'; + +const companyFeed = `SELECT c.id ${FROM} WHERE c.bidder_id = 'eik:0' ${ORDER}`; +// BEFORE denormalisation the authority feed had to scope via the tender (no authority_id on contracts). +const authorityFeedOld = `SELECT c.id ${FROM} WHERE t.authority_id = 'auth:0' ${ORDER}`; +// AFTER: scoped on the denormalised contract column, so idx_contracts_authority_recent applies. +const authorityFeedNew = `SELECT c.id ${FROM} WHERE c.authority_id = 'auth:0' ${ORDER}`; + +function seed(dbPath: string, withAuthorityId: boolean): void { + const stmts: string[] = ['BEGIN;']; + for (let i = 0; i < 20; i++) + stmts.push(`INSERT INTO authorities(id,name) VALUES('auth:${i}','A${i}');`); + for (let i = 0; i < 30; i++) + stmts.push(`INSERT INTO bidders(id,name) VALUES('eik:${i}','B${i}');`); + for (let i = 0; i < 100; i++) + stmts.push( + `INSERT INTO tenders(id,source_id,title,authority_id,cpv_code,procedure_type,status) ` + + `VALUES('t:${i}','U${i}','T${i}','auth:${i % 20}','45000000','открита','awarded');`, + ); + for (let i = 0; i < 800; i++) + stmts.push( + `INSERT INTO contracts(id,tender_id,bidder_id,amount,amount_eur,currency,value_flag,signed_at,published_at,bids_received) ` + + `VALUES('c:${i}','t:${i % 100}','eik:${i % 30}',1,1,'EUR','ok','202${i % 5}-0${(i % 9) + 1}-15','202${i % 5}-0${(i % 9) + 1}-10',1);`, + ); + stmts.push('COMMIT;'); + exec(dbPath, stmts.join('\n')); + // Mirror the ETL: populate the denormalised authority_id from the parent tender (migration 0006 does + // the same via its backfill + the UPDATE steps in normalize-raw.sql / refresh-slice.sql). + if (withAuthorityId) + exec( + dbPath, + `UPDATE contracts SET authority_id = (SELECT t.authority_id FROM tenders t WHERE t.id = contracts.tender_id);`, + ); +} + +describe('entity recent-contracts feed indexes', () => { + let dir: string; + let before: string; + let after: string; + + beforeAll(() => { + dir = mkdtempSync(resolve(tmpdir(), 'sigma-recent-feed-')); + before = resolve(dir, 'before.sqlite'); + after = resolve(dir, 'after.sqlite'); + for (const m of baseMigrations) readScript(before, m); + seed(before, false); + for (const m of baseMigrations) readScript(after, m); + readScript(after, feedMigration); + seed(after, true); + }); + + afterAll(() => rmSync(dir, { recursive: true, force: true })); + + it('company feed full-scans + temp-sorts BEFORE, index-walks AFTER', () => { + expect(plan(before, companyFeed)).toContain('USE TEMP B-TREE FOR ORDER BY'); + const p = plan(after, companyFeed); + expect(p).toContain('idx_contracts_bidder_recent'); + expect(p).not.toContain('USE TEMP B-TREE FOR ORDER BY'); + }); + + it('authority feed full-scans + temp-sorts BEFORE, index-walks AFTER (via denormalised authority_id)', () => { + expect(plan(before, authorityFeedOld)).toContain('USE TEMP B-TREE FOR ORDER BY'); + const p = plan(after, authorityFeedNew); + expect(p).toContain('idx_contracts_authority_recent'); + expect(p).not.toContain('USE TEMP B-TREE FOR ORDER BY'); + }); +}); diff --git a/packages/db/src/refresh-slice.test.ts b/packages/db/src/refresh-slice.test.ts index aa20ec76..733dfb9f 100644 --- a/packages/db/src/refresh-slice.test.ts +++ b/packages/db/src/refresh-slice.test.ts @@ -1,6 +1,6 @@ /// import { execFileSync } from 'node:child_process'; -import { mkdtempSync, rmSync } from 'node:fs'; +import { mkdtempSync, readdirSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { dirname, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -8,7 +8,9 @@ import { describe, expect, it } from 'vitest'; import { assertIntegrity } from '../../../scripts/integrity-checks.mjs'; const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); -const schemaPath = resolve(root, 'packages/db/migrations/0000_init.sql'); +// Apply the whole migration chain (not just 0000): refresh-slice.sql/normalize-raw.sql now touch the +// denormalised contracts.authority_id column added by migration 0006, so the served schema must match. +const migrationsDir = resolve(root, 'packages/db/migrations'); const refreshSlicePath = resolve(root, 'scripts/refresh-slice.sql'); const normalizePath = resolve(root, 'scripts/normalize-raw.sql'); const workStagingSchemaPath = resolve(root, 'scripts/work-staging-schema.sql'); @@ -35,6 +37,15 @@ function readScript(dbPath: string, path: string): void { }); } +// Build the served schema from the full migration chain, as production does. +function applyMigrations(dbPath: string): void { + for (const f of readdirSync(migrationsDir) + .filter((f) => f.endsWith('.sql')) + .sort()) { + readScript(dbPath, resolve(migrationsDir, f)); + } +} + function resetRawStaging(dbPath: string): void { const rows = sqliteJson<{ name: string }>( dbPath, @@ -174,7 +185,7 @@ function seedOcdsOnlySharedNumber(dbPath: string): void { } function initWorkDb(dbPath: string): void { - readScript(dbPath, schemaPath); + applyMigrations(dbPath); readScript(dbPath, workStagingSchemaPath); } @@ -357,7 +368,7 @@ describe('refresh-slice EOP base derivation', () => { const dir = mkdtempSync(resolve(tmpdir(), 'sigma-refresh-slice-')); const dbPath = resolve(dir, 'test.sqlite'); try { - readScript(dbPath, schemaPath); + applyMigrations(dbPath); readScript(dbPath, workStagingSchemaPath); seedEopBaseDay(dbPath); @@ -436,7 +447,7 @@ describe('refresh-slice EOP base derivation', () => { const dir = mkdtempSync(resolve(tmpdir(), 'sigma-refresh-slice-')); const dbPath = resolve(dir, 'test.sqlite'); try { - readScript(dbPath, schemaPath); + applyMigrations(dbPath); readScript(dbPath, workStagingSchemaPath); seedEopOnlySharedNumber(dbPath); readScript(dbPath, refreshSlicePath); @@ -484,7 +495,7 @@ describe('refresh-slice EOP base derivation', () => { const dir = mkdtempSync(resolve(tmpdir(), 'sigma-refresh-slice-')); const dbPath = resolve(dir, 'test.sqlite'); try { - readScript(dbPath, schemaPath); + applyMigrations(dbPath); readScript(dbPath, workStagingSchemaPath); sqlite( dbPath, @@ -532,7 +543,7 @@ describe('refresh-slice EOP base derivation', () => { const dir = mkdtempSync(resolve(tmpdir(), 'sigma-refresh-slice-')); const dbPath = resolve(dir, 'test.sqlite'); try { - readScript(dbPath, schemaPath); + applyMigrations(dbPath); readScript(dbPath, workStagingSchemaPath); sqlite( dbPath, diff --git a/scripts/normalize-raw.sql b/scripts/normalize-raw.sql index 17cad66e..9610ade7 100644 --- a/scripts/normalize-raw.sql +++ b/scripts/normalize-raw.sql @@ -560,6 +560,13 @@ WHERE x.bidder_key IS NOT NULL AND EXISTS (SELECT 1 FROM tenders te WHERE te.id = 't:' || x.unp) AND EXISTS (SELECT 1 FROM bidders b WHERE b.id = x.bidder_key); +-- Denormalise the parent tender's authority onto the contract row so the entity RSS feed can walk a +-- scoped index (idx_contracts_authority_recent, migration 0006) instead of temp-B-tree-sorting all of +-- an authority's contracts. tender_id is set above and tenders.id is the PK, so this is an indexed lookup. +UPDATE contracts + SET authority_id = (SELECT t.authority_id FROM tenders t WHERE t.id = contracts.tender_id) + WHERE authority_id IS NULL; + -- Reconciliation guard: the final summary reports the contracts inserted alongside the surviving -- staging candidates, so a future NOT NULL/foreign-key mismatch is visible instead of hidden by -- INSERT OR IGNORE. diff --git a/scripts/refresh-slice.sql b/scripts/refresh-slice.sql index e051a7ed..a08a184a 100644 --- a/scripts/refresh-slice.sql +++ b/scripts/refresh-slice.sql @@ -1002,6 +1002,14 @@ SET status = 'awarded' WHERE status <> 'awarded' AND EXISTS (SELECT 1 FROM raw_contracts c WHERE 't:' || c.unp = tenders.id); +-- Denormalise authority onto the touched (window) contracts so the authority RSS feed's scoped index +-- (idx_contracts_authority_recent, migration 0006) covers the just-inserted rows too — otherwise a +-- newly-signed contract would be missing from the feed until the next full import. Scoped to the +-- touched set; tenders.id is the PK, so the lookup is indexed. +UPDATE contracts + SET authority_id = (SELECT t.authority_id FROM tenders t WHERE t.id = contracts.tender_id) + WHERE id IN (SELECT id FROM refresh_touched_contracts); + -- 5) Promote window amendments into served domain history and roll touched contracts. -- @refresh-batch amendments