diff --git a/.github/workflows/daily-metrics.yml b/.github/workflows/daily-metrics.yml index 279f178..3ff51f0 100644 --- a/.github/workflows/daily-metrics.yml +++ b/.github/workflows/daily-metrics.yml @@ -70,4 +70,14 @@ jobs: UPSTASH_COUNTERS_REST_TOKEN: ${{ secrets.UPSTASH_COUNTERS_REST_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} DIGEST_ISSUE_NUMBER: ${{ steps.setup.outputs.issue_num }} + # Flagged (not fixed) in PR #87: this workflow never set VERCEL_ENV, + # so lib/upstash.ts's keyPrefix() resolved to 'dev' here while the + # live site's routes write under 'production:' (VERCEL_ENV is + # Vercel-injected at request time, not something a GitHub Actions + # runner has). Every readUsageWindow/readMcpClientDay call was + # silently reading an empty 'dev:' keyspace instead of the real + # counters - the digest could only ever report zeros. Same fix + # scripts/tenant-admin.mjs's own header comment already documents + # for laptop-shell CLI runs: export VERCEL_ENV=production explicitly. + VERCEL_ENV: production run: npx tsx scripts/daily-metrics.mjs diff --git a/.github/workflows/hot-bills.yml b/.github/workflows/hot-bills.yml new file mode 100644 index 0000000..2ea111f --- /dev/null +++ b/.github/workflows/hot-bills.yml @@ -0,0 +1,89 @@ +name: Hot-bill refresh + +# Twice-daily, refresh-ONLY pass between nightly bill syncs (audit §4 Alt B / +# §5 item 3, pipeline-audit.md): the nightly sync (sync-bills.yml, 07:30 UTC) +# structurally can't reflect a floor vote or markup that happens mid-day +# until the next morning. This job re-fetches the ~100 most-recently-updated +# Congress.gov bills and refreshes status/last_action_date/urgency for +# whichever of them are ALREADY in the corpus - decodes nothing, so a +# brand-new bill still waits for the nightly sync's decode-before-publish +# gate (scripts/hot-bills.mjs's header comment has the full reasoning). +# Zero Anthropic usage: no ANTHROPIC_API_KEY in this workflow at all. + +on: + schedule: + # 17:00 and 22:00 UTC (~1pm/6pm ET) - both inside the US legislative day, + # after floor/committee activity has had time to post to Congress.gov, + # and well clear of the 07:30 UTC nightly sync on both sides. + - cron: '0 17 * * *' + - cron: '0 22 * * *' + workflow_dispatch: + +permissions: + contents: write + +concurrency: + # Shared with sync-bills.yml/refresh-legislators.yml so this can never race + # the nightly sync's commit - same reasoning as those two workflows already + # sharing this group. + group: data-sync + cancel-in-progress: false + +jobs: + hot-refresh: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-node@v6 + # scripts/hot-bills.mjs and scripts/congress-fetch.mjs are stdlib-only + # (Node's global fetch + lib/urgency.mjs, a plain .mjs with no npm + # deps) - same "the runner's node suffices, no npm ci" posture as + # refresh-legislators.yml and scripts/verify-deploy.mjs. + with: + node-version: 24 + - name: Refresh hot bills + env: + CONGRESS_API_KEY: ${{ secrets.CONGRESS_API_KEY }} + run: node scripts/hot-bills.mjs + - name: Commit data + id: commit + # Same oravan-sync author pattern as sync-bills.yml/refresh- + # legislators.yml - the author email must map to the Vercel-linked + # GitHub account or Vercel BLOCKs the auto-deploy (docs/solutions/ + # vercel-bot-push-blocked-deploys.md). + run: | + git config user.name "oravan-sync" + git config user.email "223600121+cm2489@users.noreply.github.com" + git add data/ + if git diff --cached --quiet; then + echo "no changes" + echo "changed=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + echo "changed=true" >> "$GITHUB_OUTPUT" + git commit -m "chore(data): hot-bill refresh $(date -u +%FT%HZ)" + # This job only ever touches data/bills.json (refresh-only, no ES/ + # coverage/sync-state writes) - disjoint from sync-bills.yml's and + # refresh-legislators.yml's own file sets, but the checked-out base + # can still go stale behind either of them in the shared data-sync + # concurrency group. Rebase onto the latest main and retry, same + # pattern as the other two workflows. + for i in 1 2 3 4 5; do + if git push; then + echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + exit 0 + fi + echo "push rejected, rebasing onto latest main (attempt $i)" + git fetch origin main + git rebase origin/main + done + exit 1 + - name: Verify the deploy landed + # Same dead-man's-switch as sync-bills.yml/refresh-legislators.yml - + # poll production until it serves the SHA we just pushed. Skips with + # a notice until the PROD_URL repo variable is set. + if: steps.commit.outputs.changed == 'true' + env: + PROD_URL: ${{ vars.PROD_URL }} + EXPECT_SHA: ${{ steps.commit.outputs.sha }} + run: node scripts/verify-deploy.mjs diff --git a/.github/workflows/sync-bills.yml b/.github/workflows/sync-bills.yml index e3d0d11..6d96b3d 100644 --- a/.github/workflows/sync-bills.yml +++ b/.github/workflows/sync-bills.yml @@ -7,7 +7,7 @@ on: inputs: max_new_decodes: description: 'Max new bills to AI-decode this run' - default: '40' + default: '120' permissions: contents: write @@ -35,7 +35,7 @@ jobs: env: CONGRESS_API_KEY: ${{ secrets.CONGRESS_API_KEY }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - MAX_NEW_DECODES: ${{ inputs.max_new_decodes || '40' }} + MAX_NEW_DECODES: ${{ inputs.max_new_decodes || '120' }} run: node scripts/sync-bills.mjs - name: Backfill coverage search inputs # Drains press_names/news_query for bills decoded before search-input @@ -143,6 +143,15 @@ jobs: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} UPSTASH_CACHE_REST_URL: ${{ secrets.UPSTASH_CACHE_REST_URL }} UPSTASH_CACHE_REST_TOKEN: ${{ secrets.UPSTASH_CACHE_REST_TOKEN }} + # 2026-07-16 metrics-env fix (same root cause as PR #87's "Flagged, + # not fixed" note on daily-metrics.yml): this workflow never set + # VERCEL_ENV either, so lib/scriptcache.ts's keys were written under + # keyPrefix()'s 'dev' fallback instead of 'production' — every combo + # this step ever cached since PREGEN_ENABLED was armed (2026-07-12) + # landed in a keyspace app/api/script/route.ts's live requests + # (VERCEL_ENV=production, Vercel-injected) never read, so every + # pregen run has been pure spend with zero possible cache hits. + VERCEL_ENV: production run: | if [ "$PREGEN_ENABLED" != "true" ]; then echo "::notice::pregen SKIPPED - repo variable PREGEN_ENABLED is not 'true'. \$0 spent tonight. See the S21 PR's Arming checklist to turn this on." diff --git a/app/[locale]/bills/page.tsx b/app/[locale]/bills/page.tsx index b2e42e8..db58096 100644 --- a/app/[locale]/bills/page.tsx +++ b/app/[locale]/bills/page.tsx @@ -37,7 +37,7 @@ export default async function BillsPage({ params }: { params: Promise<{ locale: )} - + ); } diff --git a/app/[locale]/page.tsx b/app/[locale]/page.tsx index 5475b50..c77ff7e 100644 --- a/app/[locale]/page.tsx +++ b/app/[locale]/page.tsx @@ -139,7 +139,7 @@ export default async function HomePage({ params }: { params: Promise<{ locale: s ) : quiet ? (
- +
) : null} diff --git a/app/[locale]/reps/page.tsx b/app/[locale]/reps/page.tsx index 2e63200..81bfae8 100644 --- a/app/[locale]/reps/page.tsx +++ b/app/[locale]/reps/page.tsx @@ -176,7 +176,7 @@ export default async function RepsPage({ ) : (
- +
)} >>({}); @@ -195,7 +196,7 @@ export function BillsBrowser({ bills, checkedAt }: { bills: FeedTeaser[]; checke

{t(`bills.bandSub.${band}`)}

- +
); diff --git a/components/UrgencyEmptyState.tsx b/components/UrgencyEmptyState.tsx index 3d33a84..db6dc1e 100644 --- a/components/UrgencyEmptyState.tsx +++ b/components/UrgencyEmptyState.tsx @@ -2,7 +2,7 @@ import { useSyncExternalStore } from 'react'; import { useFormatter, useTranslations } from 'next-intl'; -import { emptyStateVerdict } from '@/lib/freshness-state'; +import { emptyStateVerdict, type FreshnessSignals } from '@/lib/freshness-state'; // The React-idiomatic hydration gate: server snapshot (and the hydration // render) reads false, the first client snapshot reads true. No state, no @@ -20,10 +20,13 @@ const useHydrated = () => * bills clearing the urgency floor. Which of the two honest messages shows * depends on whether the data itself is trustworthy right now: * - quiet week: floor cleared no bills, but the corpus was checked recently - * - a real quiet week, said plainly instead of backfilled from rank. - * - data stale: the last successful check is older than the claim window, - * so an empty list might just mean "we haven't looked lately" - the copy - * says that, not "quiet." + * AND the sync cursor/corpus itself shows real recent progress - a real + * quiet week, said plainly instead of backfilled from rank. + * - data stale: either the last successful check is older than its claim + * window, or the sync cursor / newest known activity has gone dark past + * the wider dead window (lib/freshness-state.ts's emptyStateVerdict has + * the full threshold reasoning) - an empty list might just mean "we + * haven't actually looked lately," and the copy says that, not "quiet." * * This must stay a client component: the site is largely static-generated, * so a server-rendered verdict freezes at build time and a dead sync would @@ -35,7 +38,7 @@ const useHydrated = () => * the visitor's: baked "quiet week" HTML would flash (or, with JS off, * permanently claim) quiet on long-dead data. */ -export function UrgencyEmptyState({ checkedAt }: { checkedAt: string }) { +export function UrgencyEmptyState({ checkedAt, completeThrough, newestAction }: FreshnessSignals) { const t = useTranslations('freshness'); const format = useFormatter(); const hydrated = useHydrated(); @@ -51,7 +54,7 @@ export function UrgencyEmptyState({ checkedAt }: { checkedAt: string }) { ); } - const staleVerdict = emptyStateVerdict(checkedAt) === 'data_stale'; + const staleVerdict = emptyStateVerdict({ checkedAt, completeThrough, newestAction }) === 'data_stale'; return (

diff --git a/lib/core/mcp.ts b/lib/core/mcp.ts index 1134299..82d7422 100644 --- a/lib/core/mcp.ts +++ b/lib/core/mcp.ts @@ -565,11 +565,14 @@ export function whatsMoving(params: WhatsMovingParams, locale: Locale) { * AE3/KTD-2 honesty rule, reusing lib/freshness-state.ts's collapse rather * than re-deriving it (that file's own doc comment names this exact * tool): an empty result reads as a genuine "quiet week" only while the - * nightly pipeline itself looks alive. A stale or dead pipeline must never - * be dressed up as "nothing to act on this week" - that would hand an - * agent a fact about our sync health disguised as a fact about Congress. + * nightly pipeline itself looks alive AND the sync cursor/corpus itself + * shows real recent progress (not just "the job executed" - see + * emptyStateVerdict's own doc comment for why both signals matter). A + * stale or dead pipeline must never be dressed up as "nothing to act on + * this week" - that would hand an agent a fact about our sync health + * disguised as a fact about Congress. */ - const verdict = limited.length === 0 ? emptyStateVerdict(getFreshness().checkedAt) : null; + const verdict = limited.length === 0 ? emptyStateVerdict(getFreshness()) : null; return { bills: limited, diff --git a/lib/freshness-state.ts b/lib/freshness-state.ts index 9d3531c..4ba1ae6 100644 --- a/lib/freshness-state.ts +++ b/lib/freshness-state.ts @@ -40,16 +40,56 @@ export function freshnessState(checkedAt: string, now: number = Date.now()): Fre export type EmptyStateVerdict = 'quiet_week' | 'data_stale'; +/** The three freshness signals `emptyStateVerdict` reads — structurally the + * same shape as lib/freshness.ts's `Freshness`, redeclared here (rather than + * imported) so this file stays free of any data import and safe to ship to + * a client bundle, per the header comment above. */ +export interface FreshnessSignals { + /** Last successful nightly sync run — "did the job run at all". */ + checkedAt: string; + /** Sync cursor high-water mark — "how far the backlog scan has actually + * processed" (data/sync-state.json's lastSync). */ + completeThrough: string; + /** Newest `last_action_date` across the whole corpus — "is there anything + * current in the data at all", regardless of what the sync's own + * bookkeeping claims. */ + newestAction: string; +} + /** * The AE3 collapse rule as an importable primitive: an empty "Act now" band - * is a quiet week only while the data is fresh; both 'stale' and 'dead' - * collapse to data_stale — never assert quiet on dead data. This is the ONE - * copy of that rule: the site's empty band renders it (components/ - * UrgencyEmptyState.tsx), and the future MCP `whats_moving` tool must import - * this same function rather than re-deriving the collapse inline — a second - * copy is the exact drift docs/solutions/stale-urgency-freeze.md closed for - * the urgency curve. + * reads as a genuine quiet_week only when EVERY freshness signal checks out. + * This is the ONE copy of that rule: the site's empty band renders it + * (components/UrgencyEmptyState.tsx), and lib/core/mcp.ts's `whatsMoving` + * imports this same function rather than re-deriving the collapse inline — + * a second copy is the exact drift docs/solutions/stale-urgency-freeze.md + * closed for the urgency curve. + * + * Two different thresholds, deliberately (2026-07-16, audit §5 item 4): + * - `checkedAt` (did the nightly job even run tonight) uses the tight + * FRESHNESS_CLAIM_WINDOW_DAYS/FRESHNESS_DEAD_WINDOW_DAYS pair via + * freshnessState — a "we checked recently" claim should go stale fast. + * - `completeThrough` (the sync cursor) and `newestAction` (the corpus's + * own newest activity) instead trip data_stale only past + * FRESHNESS_DEAD_WINDOW_DAYS, the wider of the two constants. Both are + * EXPECTED to lag `checkedAt` by real days under ordinary operation — the + * ascending backlog-scan cursor deliberately trails while it drains (see + * lib/freshness.ts's own doc comment and scripts/sync-bills.mjs's + * two-pass fetch design note) — so gating them on the tight claim window + * would make the site cry "data stale" every single night even when + * tonight's recent-first pass kept the actually-relevant content current. + * The wide dead window instead catches the failure mode this item exists + * for: a pipeline that runs every night, commits every night, and reports + * itself "fresh" via `checkedAt` alone while making no real forward + * progress for weeks (the bug this audit found — a 29-day-old cursor and + * a 29-day-old newest action, both silently passing as "fresh" under the + * old checkedAt-only check). Past three weeks with NOTHING new anywhere + * in the corpus, "quiet week" stops being a credible claim regardless of + * how recently the job merely executed. */ -export function emptyStateVerdict(checkedAt: string, now: number = Date.now()): EmptyStateVerdict { - return freshnessState(checkedAt, now) === 'fresh' ? 'quiet_week' : 'data_stale'; +export function emptyStateVerdict(signals: FreshnessSignals, now: number = Date.now()): EmptyStateVerdict { + if (freshnessState(signals.checkedAt, now) !== 'fresh') return 'data_stale'; + if (freshnessAgeDays(signals.completeThrough, now) > FRESHNESS_DEAD_WINDOW_DAYS) return 'data_stale'; + if (freshnessAgeDays(signals.newestAction, now) > FRESHNESS_DEAD_WINDOW_DAYS) return 'data_stale'; + return 'quiet_week'; } diff --git a/scripts/congress-fetch.mjs b/scripts/congress-fetch.mjs new file mode 100644 index 0000000..62d9afb --- /dev/null +++ b/scripts/congress-fetch.mjs @@ -0,0 +1,152 @@ +/** + * Congress.gov fetch + status/urgency/category-mapping helpers shared by the + * nightly bill sync (scripts/sync-bills.mjs: fetch + AI decode + commit) and + * the twice-daily hot-bill refresh (scripts/hot-bills.mjs: fetch + refresh + * only, zero AI cost) - extracted 2026-07-16 (audit §4 Alt B / §5 item 3) so + * the two scripts share one implementation of "talk to Congress.gov" and + * "map a bill-detail payload onto our fields" instead of maintaining two + * copies that can drift (the same "one copy" discipline lib/urgency.mjs's + * own doc comment already applies to the urgency curve). + * + * Needs CONGRESS_API_KEY in the importing process's env. + */ +import { STATUS_BASE } from '../lib/urgency.mjs'; + +export const CONGRESS = 119; +export const BILL_TYPES = new Set(['hr', 's', 'hjres', 'sjres']); + +const API = 'https://api.congress.gov/v3'; +const KEY = process.env.CONGRESS_API_KEY; +if (!KEY) throw new Error('CONGRESS_API_KEY missing'); + +/** GET one Congress.gov endpoint, retrying on a bad status or a thrown/timed + * out request (a hung socket must retry, not kill the whole run - the + * 2026-06-13 crash). */ +export async function cg(path, params = {}) { + const url = new URL(`${API}${path}`); + for (const [k, v] of Object.entries(params)) url.searchParams.set(k, v); + url.searchParams.set('api_key', KEY); + url.searchParams.set('format', 'json'); + let lastErr; + for (let attempt = 0; attempt <= 4; attempt++) { + if (attempt > 0) await new Promise((r) => setTimeout(r, 2000 * attempt)); + try { + // 30s per-request ceiling: a hung socket fails fast and retries instead + // of hanging on undici's ~5min headers timeout (the 2026-06-13 crash). + const res = await fetch(url, { signal: AbortSignal.timeout(30_000) }); + // await inside the try: the 30s abort can fire mid-body-read, and an + // un-awaited res.json() rejection would escape the catch and kill the + // run uncaught instead of retrying (the 2026-07-04 crash). + if (res.ok) return await res.json(); + lastErr = new Error(`Congress.gov ${res.status} for ${path}`); + } catch (e) { + lastErr = e; // network error / timeout - retry rather than kill the run + } + } + throw lastErr; +} + +/** The N most-recently-updated bills across the whole corpus (no fromDateTime + * floor - literally "what changed most recently"), of our 4 tracked types. + * Used by sync-bills.mjs's recent-first pass and the whole of hot-bills.mjs; + * see the audit's two-pass fetch design (§5 item 2 / §4 Alt B): the + * ascending "since cursor" scan structurally reaches the newest bills LAST, + * so both freshness-sensitive callers fetch this descending window instead. */ +export async function fetchRecentlyUpdated(limit) { + const page = await cg(`/bill/${CONGRESS}`, { sort: 'updateDate+desc', limit }); + const items = page.bills ?? []; + return items.filter((b) => BILL_TYPES.has((b.type ?? '').toLowerCase())); +} + +// ---- status mapping (ported from the reference implementation) ---- +export function mapStatus(actionText) { + const text = (actionText ?? '').toLowerCase().trim(); + if (!text) return 'committee'; + if (text.includes('became public law') || text.includes('signed by president')) return 'signed'; + if (text.includes('vetoed')) return 'vetoed'; + if (text.includes('conference report') || text.includes('conference committee')) return 'conference'; + if ( + text.includes('passed house') || text.includes('passed senate') || + text.includes('passed/agreed to') || text.includes('agreed to in') || + text.includes('received in the senate') || text.includes('received in the house') || + text.includes('held at the desk') + ) return 'passed_chamber'; + if ( + text.includes('placed on') || text.includes('calendar') || + text.includes('cloture') || text.includes('rule provid') || + text.includes('motion to proceed') + ) return 'floor_vote'; + if (text.includes('markup') || text.includes('ordered to be reported') || text.includes('reported by')) return 'markup'; + return 'committee'; +} + +// Stored sync-time score (freshness bonus, no decay) - the FEED never ranks +// by this; read-time effectiveUrgency in lib/urgency.mjs does the ranking. +// The base table is shared so the two curves can't drift apart. +export function urgencyScore(status, lastActionDate) { + const base = STATUS_BASE[status] ?? 0.2; + let bonus = 0; + if (lastActionDate) { + const days = (Date.now() - new Date(lastActionDate).getTime()) / 86_400_000; + if (Number.isFinite(days)) bonus = days < 3 ? 0.1 : days < 7 ? 0.05 : 0; + } + return Math.round(Math.min(1, Math.max(0, base + bonus)) * 1000) / 1000; +} + +// CRS Policy Area -> our 12 flat categories (1:1, all 32 areas covered) +const POLICY_AREA_TO_CATEGORY = { + 'Labor and Employment': 'jobs_economy', 'Commerce': 'jobs_economy', + 'Finance and Financial Sector': 'jobs_economy', 'Taxation': 'jobs_economy', + 'Economics and Public Finance': 'jobs_economy', 'Agriculture and Food': 'jobs_economy', + 'Transportation and Public Works': 'jobs_economy', + 'Science, Technology, Communications': 'ai_technology', + 'Health': 'health', + 'Housing and Community Development': 'housing', + 'Immigration': 'immigration', + 'Government Operations and Politics': 'government_democracy', 'Congress': 'government_democracy', + 'Emergency Management': 'government_democracy', + 'Crime and Law Enforcement': 'crime_justice', 'Law': 'crime_justice', + 'Education': 'education', 'Sports and Recreation': 'education', + 'Social Sciences and History': 'education', + 'Environmental Protection': 'environment_energy', 'Energy': 'environment_energy', + 'Public Lands and Natural Resources': 'environment_energy', + 'Water Resources Development': 'environment_energy', 'Animals': 'environment_energy', + 'Civil Rights and Liberties, Minority Issues': 'rights_liberties', + 'Armed Forces and National Security': 'national_security', + 'International Affairs': 'national_security', + 'Foreign Trade and International Finance': 'national_security', + 'Families': 'family_community', 'Social Welfare': 'family_community', + 'Native Americans': 'family_community', 'Arts, Culture, Religion': 'family_community', +}; + +export function tagBill(policyArea) { + const cat = POLICY_AREA_TO_CATEGORY[policyArea ?? '']; + return cat ? [cat] : []; +} + +export function slugOf(b) { + return `${b.bill_type}-${b.bill_number}-${b.congress_number}`.toLowerCase(); +} + +/** Slug for a Congress.gov bill-list item ({type, number}), not yet a corpus + * bill object - the shape sync-bills.mjs's `updated`/recent-pass arrays and + * hot-bills.mjs's fetch results are in. */ +export function updateSlug(u, congress = CONGRESS) { + return `${u.type.toLowerCase()}-${u.number}-${congress}`.toLowerCase(); +} + +/** Mutate an existing corpus bill's refreshable fields in place from a + * Congress.gov bill-detail payload (`cg('/bill/{congress}/{type}/{number}')`'s + * `.bill`). Free, no AI cost - the one place both scripts' "refresh" branch + * lives, so it can't drift between the nightly sync and the hot-bill pass. */ +export function refreshBillFields(existing, detail) { + const status = mapStatus(detail.latestAction?.text); + const lastActionDate = detail.latestAction?.actionDate ?? null; + existing.status = status; + existing.last_action_date = lastActionDate; + existing.last_action_text = detail.latestAction?.text ?? existing.last_action_text; + existing.urgency_score = urgencyScore(status, lastActionDate); + const tags = tagBill(detail.policyArea?.name); + if (tags.length) existing.issue_tags = tags; + existing.policy_area = detail.policyArea?.name ?? existing.policy_area; +} diff --git a/scripts/hot-bills.mjs b/scripts/hot-bills.mjs new file mode 100644 index 0000000..fab6864 --- /dev/null +++ b/scripts/hot-bills.mjs @@ -0,0 +1,66 @@ +/** + * Twice-daily hot-bill refresh (audit §4 Alt B / §5 item 3). REFRESH-ONLY: + * updates status/last_action_date/last_action_text/urgency_score/issue_tags + * for bills ALREADY in data/bills.json, using the same "~100 most-recently- + * updated across the whole 119th Congress" Congress.gov window + * scripts/sync-bills.mjs's recent-first pass uses (scripts/congress-fetch.mjs, + * shared so the two scripts can't drift). + * + * node --env-file=.env.local scripts/hot-bills.mjs + * + * Decodes NOTHING. A brand-new bill discovered here is left for the nightly + * sync (scripts/sync-bills.mjs) - that script is the only place the + * decode-before-publish gate runs, so this script must never publish an + * undecoded bill. Zero Anthropic usage: needs only CONGRESS_API_KEY. This is + * the tradeoff named up front in the audit's Alt B - a bill that's brand new + * AND breaking mid-day still waits until the next 07:30 UTC nightly sync to + * actually appear on the site; only bills already in the corpus get same-day + * status/urgency freshness from this pass. + * + * Runs 2x/day (.github/workflows/hot-bills.yml, 17:00 + 22:00 UTC - inside + * the US legislative day) between nightly syncs, so a floor vote or markup + * that happens mid-day is reflected in effectiveUrgency (lib/urgency.mjs) + * same-day instead of sitting stale until the next morning's sync. Also + * directly improves lib/freshness.ts's `newestAction` signal (scanned live + * from data/bills.json's last_action_date values), independent of + * data/sync-state.json - this script intentionally never touches + * sync-state.json; that file's lastRun/lastSync are the NIGHTLY sync's own + * "did the job run" / "how far has the backlog scan processed" signals, and + * conflating a same-day refresh pass with the sync cursor would work against + * lib/freshness-state.ts's honesty model, not for it. + */ +import { readFileSync, writeFileSync } from 'node:fs'; +import { CONGRESS, cg, fetchRecentlyUpdated, refreshBillFields, slugOf, updateSlug } from './congress-fetch.mjs'; + +const FETCH_LIMIT = Number(process.env.HOT_BILLS_FETCH_LIMIT ?? 100); + +const bills = JSON.parse(readFileSync('data/bills.json', 'utf8')); +const bySlug = new Map(bills.map((b) => [slugOf(b), b])); + +console.log(`hot-bill refresh: fetching up to ${FETCH_LIMIT} most-recently-updated bills`); +const recent = await fetchRecentlyUpdated(FETCH_LIMIT); + +let refreshed = 0, newSkipped = 0, failed = 0; +for (const u of recent) { + const type = u.type.toLowerCase(); + const slug = updateSlug(u); + const existing = bySlug.get(slug); + if (!existing) { + newSkipped++; // brand-new bill - decode-before-publish waits for the nightly sync + continue; + } + try { + const { bill: d } = await cg(`/bill/${CONGRESS}/${type}/${u.number}`); + refreshBillFields(existing, d); + refreshed++; + } catch (e) { + failed++; + console.error(`FAIL ${slug}: ${e.message}`); + } +} + +writeFileSync('data/bills.json', JSON.stringify(bills)); +console.log( + `DONE: ${refreshed} refreshed, ${newSkipped} new bill(s) skipped (nightly sync decodes those), ${failed} failed; corpus ${bills.length}` +); +if (failed > recent.length / 2) process.exit(1); // mostly-failed run: don't let CI commit garbage diff --git a/scripts/sync-bills.mjs b/scripts/sync-bills.mjs index d1c552e..8dd8c02 100644 --- a/scripts/sync-bills.mjs +++ b/scripts/sync-bills.mjs @@ -12,19 +12,69 @@ * their EN+ES summary and headline exist, so the feed never shows * undecoded entries. At most MAX_NEW_DECODES per run (cost ceiling); * the rest wait for the next night. + * + * Two-pass fetch (2026-07-16, audit §5 item 2). Congress.gov is queried + * TWICE per run, in this order: + * 1. Recent-first: `sort=updateDate+desc, limit=RECENT_FETCH_LIMIT` - the + * ~100 most-recently-touched bills in the whole 119th Congress, no + * cursor floor. Already-known bills refresh for free; brand-new bills + * decode within a RESERVED sub-budget (RECENT_DECODE_RESERVE, carved + * OUT of MAX_NEW_DECODES, not additional). This exists because the + * ascending backlog scan below structurally reaches the newest bills + * LAST - on a night with a deep backlog (or a busy legislative day) a + * floor vote that just happened would otherwise lose the race against + * both MAX_UPDATES and MAX_NEW_DECODES every single night, which is + * exactly how HR 7378 (and the whole "worth a call" feed) went stale + * for weeks even on clean, successful runs (see the audit). + * 2. Ascending backlog: `fromDateTime: lastSync, sort=updateDate+asc` - + * unchanged from before, drains the historical backlog oldest-first + * with whatever decode budget the recent-first pass didn't use. A bill + * already handled by pass 1 this run is skipped here (deduped, not + * re-fetched or re-decoded). + * + * CURSOR SEMANTICS (load-bearing, KTD-pinned): `state.lastSync`'s freeze- + * on-incomplete-work high-water mark is advanced ONLY by the ascending pass + * below. The recent-first pass never reads or writes `cursor`/`frozen` - it + * can find and decode a bill from last week while the ascending backlog is + * still stuck in May, and the cursor must keep meaning "the backlog scan has + * fully processed through here", not silently jump forward just because a + * recent bill happened to get handled out of order. See + * docs/solutions/pinned-sync-cursor.md for why an all-or-nothing cursor is + * exactly the failure this preserves the fix for. */ import Anthropic from '@anthropic-ai/sdk'; import { readFileSync, writeFileSync } from 'node:fs'; -import { STATUS_BASE } from '../lib/urgency.mjs'; +import { + BILL_TYPES, + CONGRESS, + cg, + fetchRecentlyUpdated, + mapStatus, + refreshBillFields, + slugOf, + tagBill, + updateSlug, + urgencyScore, +} from './congress-fetch.mjs'; import { generateSearchInputs } from './search-inputs.mjs'; -const CONGRESS = 119; -const BILL_TYPES = new Set(['hr', 's', 'hjres', 'sjres']); const MAX_UPDATES = Number(process.env.MAX_UPDATES ?? 500); -const MAX_NEW_DECODES = Number(process.env.MAX_NEW_DECODES ?? 40); -const API = 'https://api.congress.gov/v3'; -const KEY = process.env.CONGRESS_API_KEY; -if (!KEY) throw new Error('CONGRESS_API_KEY missing'); +// Raised 40 -> 120 (2026-07-16, audit §5 item 1): live nightly logs showed +// 373-418 bills/night needing decode against a 40-bill budget, pinning the +// ascending-pass cursor (state.lastSync) weeks behind and starving newer +// bills of decode slots night after night. 120 doesn't fully clear that +// inflow alone (~$8-14/night at $0.07-0.15/bill) - see the two-pass fetch +// design note above for the fix that stops recency from losing the race +// structurally, independent of how large the budget is. +const MAX_NEW_DECODES = Number(process.env.MAX_NEW_DECODES ?? 120); +// The recent-first pass's fetch window (audit §5 item 2 / §4 Alt A) - same +// rough size as the twice-daily hot-bills.mjs refresh pass. +const RECENT_FETCH_LIMIT = Number(process.env.RECENT_FETCH_LIMIT ?? 100); +// New-bill decode budget RESERVED for the recent-first pass, carved out of +// (not additional to) MAX_NEW_DECODES - a night with zero brand-new bills in +// the last ~100 updates leaves the full MAX_NEW_DECODES for the ascending +// backlog pass; a night with several leaves proportionally less. +const RECENT_DECODE_RESERVE = Number(process.env.RECENT_DECODE_RESERVE ?? 20); const anthropic = new Anthropic({ maxRetries: 8 }); // Sonnet 5's tokenizer runs ~30% more tokens than 4.6 for the same text, so @@ -38,10 +88,6 @@ const es = JSON.parse(readFileSync('data/bills-es.json', 'utf8')); const state = JSON.parse(readFileSync('data/sync-state.json', 'utf8')); const bySlug = new Map(bills.map((b) => [slugOf(b), b])); -function slugOf(b) { - return `${b.bill_type}-${b.bill_number}-${b.congress_number}`.toLowerCase(); -} - // Congress.gov's bill-list `updateDate` field is date-only (e.g. "2026-06-04"), // not a full timestamp. Persisting it as-is breaks the next run's fromDateTime // query, which Congress.gov 400s on - the 2026-06-25/07-01 outage. Always @@ -50,96 +96,6 @@ function toISODateTime(d) { return /T/.test(d) ? d : `${d}T00:00:00Z`; } -async function cg(path, params = {}) { - const url = new URL(`${API}${path}`); - for (const [k, v] of Object.entries(params)) url.searchParams.set(k, v); - url.searchParams.set('api_key', KEY); - url.searchParams.set('format', 'json'); - let lastErr; - for (let attempt = 0; attempt <= 4; attempt++) { - if (attempt > 0) await new Promise((r) => setTimeout(r, 2000 * attempt)); - try { - // 30s per-request ceiling: a hung socket fails fast and retries instead - // of hanging on undici's ~5min headers timeout (the 2026-06-13 crash). - const res = await fetch(url, { signal: AbortSignal.timeout(30_000) }); - // await inside the try: the 30s abort can fire mid-body-read, and an - // un-awaited res.json() rejection would escape the catch and kill the - // run uncaught instead of retrying (the 2026-07-04 crash). - if (res.ok) return await res.json(); - lastErr = new Error(`Congress.gov ${res.status} for ${path}`); - } catch (e) { - lastErr = e; // network error / timeout - retry rather than kill the run - } - } - throw lastErr; -} - -// ---- status mapping (ported from the reference implementation) ---- -function mapStatus(actionText) { - const text = (actionText ?? '').toLowerCase().trim(); - if (!text) return 'committee'; - if (text.includes('became public law') || text.includes('signed by president')) return 'signed'; - if (text.includes('vetoed')) return 'vetoed'; - if (text.includes('conference report') || text.includes('conference committee')) return 'conference'; - if ( - text.includes('passed house') || text.includes('passed senate') || - text.includes('passed/agreed to') || text.includes('agreed to in') || - text.includes('received in the senate') || text.includes('received in the house') || - text.includes('held at the desk') - ) return 'passed_chamber'; - if ( - text.includes('placed on') || text.includes('calendar') || - text.includes('cloture') || text.includes('rule provid') || - text.includes('motion to proceed') - ) return 'floor_vote'; - if (text.includes('markup') || text.includes('ordered to be reported') || text.includes('reported by')) return 'markup'; - return 'committee'; -} - -// Stored sync-time score (freshness bonus, no decay) - the FEED never ranks -// by this; read-time effectiveUrgency in lib/urgency.mjs does the ranking. -// The base table is shared so the two curves can't drift apart. -function urgencyScore(status, lastActionDate) { - const base = STATUS_BASE[status] ?? 0.2; - let bonus = 0; - if (lastActionDate) { - const days = (Date.now() - new Date(lastActionDate).getTime()) / 86_400_000; - if (Number.isFinite(days)) bonus = days < 3 ? 0.1 : days < 7 ? 0.05 : 0; - } - return Math.round(Math.min(1, Math.max(0, base + bonus)) * 1000) / 1000; -} - -// CRS Policy Area -> our 12 flat categories (1:1, all 32 areas covered) -const POLICY_AREA_TO_CATEGORY = { - 'Labor and Employment': 'jobs_economy', 'Commerce': 'jobs_economy', - 'Finance and Financial Sector': 'jobs_economy', 'Taxation': 'jobs_economy', - 'Economics and Public Finance': 'jobs_economy', 'Agriculture and Food': 'jobs_economy', - 'Transportation and Public Works': 'jobs_economy', - 'Science, Technology, Communications': 'ai_technology', - 'Health': 'health', - 'Housing and Community Development': 'housing', - 'Immigration': 'immigration', - 'Government Operations and Politics': 'government_democracy', 'Congress': 'government_democracy', - 'Emergency Management': 'government_democracy', - 'Crime and Law Enforcement': 'crime_justice', 'Law': 'crime_justice', - 'Education': 'education', 'Sports and Recreation': 'education', - 'Social Sciences and History': 'education', - 'Environmental Protection': 'environment_energy', 'Energy': 'environment_energy', - 'Public Lands and Natural Resources': 'environment_energy', - 'Water Resources Development': 'environment_energy', 'Animals': 'environment_energy', - 'Civil Rights and Liberties, Minority Issues': 'rights_liberties', - 'Armed Forces and National Security': 'national_security', - 'International Affairs': 'national_security', - 'Foreign Trade and International Finance': 'national_security', - 'Families': 'family_community', 'Social Welfare': 'family_community', - 'Native Americans': 'family_community', 'Arts, Culture, Religion': 'family_community', -}; - -function tagBill(policyArea) { - const cat = POLICY_AREA_TO_CATEGORY[policyArea ?? '']; - return cat ? [cat] : []; -} - // ---- AI decode (new bills only) ---- async function fetchBillText(type, number) { const data = await cg(`/bill/${CONGRESS}/${type}/${number}/text`); @@ -258,6 +214,110 @@ const since = state.lastSync; const runStart = new Date().toISOString(); console.log(`sync since ${since}`); +// Shared new-bill decode-budget counter - both passes below decrement into +// this ONE pool (RECENT_DECODE_RESERVE is a ceiling on the recent-first +// pass's share of it, not a separate allowance; see the header comment). +let added = 0; +let refreshed = 0; // combined total across both passes (log-only, not gated) + +/** + * Fetch one bill's current detail and either refresh it (already in the + * corpus - free) or decode it as new (only if `allowDecode`). The one place + * both passes below do "turn a Congress.gov update item into a corpus + * mutation", so the decode-before-publish invariant and the refresh fields + * can't drift between the recent-first pass and the ascending backlog pass. + * Returns one of: + * 'refreshed' - an existing bill's fields were updated in place + * 'added' - a brand-new bill was decoded and pushed into the corpus + * 'budget' - a brand-new bill was found but `allowDecode` was false + * 'failed' - the fetch or decode threw; `isNew` tells the caller + * whether this was a new-bill decode failure (must retry) + * or an existing bill's transient refresh failure + * (idempotent, self-heals on its next update). + */ +async function syncOneBill(u, allowDecode) { + const type = u.type.toLowerCase(); + const slug = updateSlug(u); + try { + const { bill: d } = await cg(`/bill/${CONGRESS}/${type}/${u.number}`); + const existing = bySlug.get(slug); + if (existing) { + refreshBillFields(existing, d); + return { outcome: 'refreshed', slug }; + } + if (!allowDecode) return { outcome: 'budget', slug }; + const status = mapStatus(d.latestAction?.text); + const lastActionDate = d.latestAction?.actionDate ?? null; + const bill = { + full_identifier: slug, + congress_number: CONGRESS, + bill_type: type, + bill_number: Number(u.number), + title: d.title, + short_title: null, + ai_summary: null, ai_headline: null, + sponsor_bioguide_id: d.sponsors?.[0]?.bioguideId ?? null, + introduced_date: d.introducedDate ?? null, + last_action_date: lastActionDate, + last_action_text: d.latestAction?.text ?? null, + status, + issue_tags: tagBill(d.policyArea?.name), + policy_area: d.policyArea?.name ?? null, + urgency_score: urgencyScore(status, lastActionDate), + congress_gov_url: `https://www.congress.gov/bill/${CONGRESS}th-congress/${type === 'hr' ? 'house-bill' : type === 's' ? 'senate-bill' : type === 'hjres' ? 'house-joint-resolution' : 'senate-joint-resolution'}/${u.number}`, + }; + const text = await fetchBillText(type, u.number); + const dec = await decode(bill, text); + bill.ai_summary = dec.ai_summary; + bill.ai_headline = dec.ai_headline; + bill.ai_sections = dec.ai_sections; + // Search handles for the coverage sync (press names + subject query). + // Non-fatal: the backfill script sweeps up any misses. + try { + const si = await generateSearchInputs(anthropic, bill); + bill.press_names = si.press_names; + bill.news_query = si.news_query; + } catch (e) { + console.error(` search-inputs failed for ${slug}: ${e.message}`); + } + es[slug] = { headline: dec.es_headline, summary: dec.es_summary, sections: dec.es_sections }; + bills.push(bill); + bySlug.set(slug, bill); + return { outcome: 'added', slug }; + } catch (e) { + console.error(`FAIL ${slug}: ${e.message}`); + return { outcome: 'failed', slug, isNew: !bySlug.has(slug) }; + } +} + +// ---- Pass 1: recent-first (audit §5 item 2) ---------------------------- +// Guarantees this run always sees the most recently-touched bills in +// Congress, no matter how deep the ascending backlog is. `handledSlugs` +// tracks everything this pass successfully resolved so pass 2 can dedupe +// without re-fetching or re-decoding - see updateSlug/refreshBillFields. +const handledSlugs = new Set(); +const recentDecodeCap = Math.min(RECENT_DECODE_RESERVE, MAX_NEW_DECODES); +console.log(`recent-first pass: fetching up to ${RECENT_FETCH_LIMIT} most-recently-updated bills (decode reserve ${recentDecodeCap})`); +const recentBills = await fetchRecentlyUpdated(RECENT_FETCH_LIMIT); +let recentRefreshed = 0, recentAdded = 0, recentDeferred = 0, recentFailed = 0; +for (const u of recentBills) { + const result = await syncOneBill(u, added < recentDecodeCap); + if (result.outcome === 'refreshed') { + refreshed++; recentRefreshed++; handledSlugs.add(result.slug); + } else if (result.outcome === 'added') { + added++; recentAdded++; handledSlugs.add(result.slug); + } else if (result.outcome === 'budget') { + recentDeferred++; // new bill, reserve exhausted - left for pass 2 (same run) or next run + } else { + recentFailed++; // logged only; deliberately NOT folded into the abort check below + } +} +console.log(`recent-first pass: ${recentRefreshed} refreshed, ${recentAdded} added+decoded, ${recentDeferred} deferred (reserve exhausted), ${recentFailed} failed`); + +// ---- Pass 2: ascending backlog scan from the cursor --------------------- +// Unchanged shape from before the two-pass fetch - see the header comment. +// The freeze-on-incomplete-work cursor logic below is tied ONLY to this +// pass; pass 1 above never touches `cursor`/`frozen`. const updated = []; let offset = 0; for (;;) { @@ -271,7 +331,7 @@ for (;;) { } console.log(`${updated.length} updated bills (capped at ${MAX_UPDATES})`); -let refreshed = 0, added = 0, queued = 0, failed = 0; +let queued = 0, failed = 0; // High-water mark: advance the cursor over every bill we fully handle, and // freeze it the instant we hit one that still needs work (decode budget // exhausted, or a new bill whose decode failed). A transient *refresh* failure @@ -281,70 +341,27 @@ let refreshed = 0, added = 0, queued = 0, failed = 0; let cursor = since; let frozen = false; for (const u of updated.slice(0, MAX_UPDATES)) { - const type = u.type.toLowerCase(); - const slug = `${type}-${u.number}-${CONGRESS}`; + const slug = updateSlug(u); let needsWork = false; - try { - const { bill: d } = await cg(`/bill/${CONGRESS}/${type}/${u.number}`); - const status = mapStatus(d.latestAction?.text); - const lastActionDate = d.latestAction?.actionDate ?? null; - const existing = bySlug.get(slug); - if (existing) { - existing.status = status; - existing.last_action_date = lastActionDate; - existing.last_action_text = d.latestAction?.text ?? existing.last_action_text; - existing.urgency_score = urgencyScore(status, lastActionDate); - const tags = tagBill(d.policyArea?.name); - if (tags.length) existing.issue_tags = tags; - existing.policy_area = d.policyArea?.name ?? existing.policy_area; + if (handledSlugs.has(slug)) { + // Already fully resolved by the recent-first pass this run - dedupe, + // don't re-fetch/re-decode. Resolved is resolved, so the cursor may + // still advance over it exactly as if pass 2 had handled it itself. + } else { + const result = await syncOneBill(u, added < MAX_NEW_DECODES); + if (result.outcome === 'refreshed') { refreshed++; - } else if (added < MAX_NEW_DECODES) { - const bill = { - full_identifier: slug, - congress_number: CONGRESS, - bill_type: type, - bill_number: Number(u.number), - title: d.title, - short_title: null, - ai_summary: null, ai_headline: null, - sponsor_bioguide_id: d.sponsors?.[0]?.bioguideId ?? null, - introduced_date: d.introducedDate ?? null, - last_action_date: lastActionDate, - last_action_text: d.latestAction?.text ?? null, - status, - issue_tags: tagBill(d.policyArea?.name), - policy_area: d.policyArea?.name ?? null, - urgency_score: urgencyScore(status, lastActionDate), - congress_gov_url: `https://www.congress.gov/bill/${CONGRESS}th-congress/${type === 'hr' ? 'house-bill' : type === 's' ? 'senate-bill' : type === 'hjres' ? 'house-joint-resolution' : 'senate-joint-resolution'}/${u.number}`, - }; - const text = await fetchBillText(type, u.number); - const dec = await decode(bill, text); - bill.ai_summary = dec.ai_summary; - bill.ai_headline = dec.ai_headline; - bill.ai_sections = dec.ai_sections; - // Search handles for the coverage sync (press names + subject query). - // Non-fatal: the backfill script sweeps up any misses. - try { - const si = await generateSearchInputs(anthropic, bill); - bill.press_names = si.press_names; - bill.news_query = si.news_query; - } catch (e) { - console.error(` search-inputs failed for ${slug}: ${e.message}`); - } - es[slug] = { headline: dec.es_headline, summary: dec.es_summary, sections: dec.es_sections }; - bills.push(bill); - bySlug.set(slug, bill); + } else if (result.outcome === 'added') { added++; - } else { + } else if (result.outcome === 'budget') { queued++; // decode budget exhausted; revisit next run needsWork = true; + } else { + failed++; + // A new bill that failed to decode must be retried; a failed refresh of + // a known bill is idempotent and re-touches on its next update. + if (result.isNew) needsWork = true; } - } catch (e) { - failed++; - console.error(`FAIL ${slug}: ${e.message}`); - // A new bill that failed to decode must be retried; a failed refresh of a - // known bill is idempotent and re-touches on its next update. - if (!bySlug.has(slug)) needsWork = true; } if (needsWork) frozen = true; else if (!frozen && u.updateDate) cursor = toISODateTime(u.updateDate); @@ -360,4 +377,8 @@ writeFileSync('data/bills.json', JSON.stringify(bills)); writeFileSync('data/bills-es.json', JSON.stringify(es)); writeFileSync('data/sync-state.json', JSON.stringify(state, null, 2)); console.log(`DONE: ${refreshed} refreshed, ${added} added+decoded, ${queued} queued for next run, ${failed} failed; corpus ${bills.length}`); -if (failed > updated.length / 2) process.exit(1); // mostly-failed run: don't let CI commit garbage +// Mostly-failed run: don't let CI commit garbage. Scoped to the ascending +// pass's own failed/updated.length exactly as before the two-pass fetch - +// the recent-first pass's (much smaller, logged-separately) failures don't +// feed this check. +if (failed > updated.length / 2) process.exit(1); diff --git a/scripts/sync-coverage.mjs b/scripts/sync-coverage.mjs index 02d5abc..233cfc2 100644 --- a/scripts/sync-coverage.mjs +++ b/scripts/sync-coverage.mjs @@ -24,10 +24,19 @@ if (!NEWS_API_KEY) { process.exit(0); } -// Cover every eligible bill by default (the news API's daily quota is the real -// ceiling; the run stops early and commits what it has if quota is hit). 25 -// candidates/bill is TheNewsAPI's Basic-tier per-request max. -const TOP_N = Number(process.env.COVERAGE_TOP_N ?? Infinity); +// Capped at the 150 most urgent eligible bills by default (raised from the +// prior unbounded Infinity, 2026-07-16, audit §5 item 1): at Infinity this +// queried every eligible bill in urgency order every night, so a bill sitting +// near the bottom of the ranking was effectively guaranteed to lose the race +// against TheNewsAPI's daily quota before it was ever reached - the same +// bills, night after night, never got a fresh look. 150 keeps the nightly +// query bounded while comfortably covering everything that could plausibly +// clear the "Act now"/"Moving" floors (taxonomy.ts's BAND_SIZES is 18 total), +// still overridable via env for a one-off wider sweep. The news API's daily +// quota remains the real ceiling either way - the run stops early and commits +// what it has if quota is hit. 25 candidates/bill is TheNewsAPI's Basic-tier +// per-request max. +const TOP_N = Number(process.env.COVERAGE_TOP_N ?? 150); const PER_BILL = Number(process.env.COVERAGE_PER_BILL ?? 5); const MAX_CANDIDATES = Number(process.env.COVERAGE_MAX_CANDIDATES ?? 25); const NEWS_API = 'https://api.thenewsapi.com/v1/news/all'; diff --git a/scripts/verify-sync.mjs b/scripts/verify-sync.mjs index c787a60..27a2f2f 100644 --- a/scripts/verify-sync.mjs +++ b/scripts/verify-sync.mjs @@ -11,21 +11,38 @@ * (RUN_STARTED_AT, captured by the workflow before the sync step) * - lastSync is not a full ISO-8601 datetime — the bare-date cursor that * 400-looped every night from 2026-06-25 to 07-01 (PR #16) + * - the sync cursor (lastSync) is more than CURSOR_MAX_AGE_DAYS old — see + * the "Cursor-age threshold" note below (2026-07-16, audit §5 item 4; + * promoted from a non-blocking ::warning) * - the bill count dropped more than 2% vs the committed corpus (the sync * only ever appends, so any real drop means corruption) * - EN/ES parity broke: a decoded bill without a bills-es.json entry, or * an ES entry pointing at a bill that doesn't exist * - * WARNS (::warning, never fails) on corpus staleness: the cursor is weeks - * behind BY DESIGN while the 361-bill decode backlog drains (the high-water + * Cursor-age threshold (2026-07-16, audit §5 item 4). This check used to be + * a non-blocking ::warning, on the theory that the cursor would sit weeks + * behind BY DESIGN while a 361-bill decode backlog drained (the high-water * mark freezes at the oldest bill still awaiting decode — see - * docs/solutions/pinned-sync-cursor.md). A wall-clock staleness failure - * would fire every night until the backlog clears; revisit the threshold - * once these warnings stop. + * docs/solutions/pinned-sync-cursor.md). Live logs proved that premise + * false: the warning fired every clean night for weeks (06-16 through + * 07-14) and was never acted on — exactly the silent-failure shape this + * script exists to prevent, and the root cause behind "worth a call" + * reading stale/empty in production. Promoted to a hard failure, at a + * DELIBERATELY GENEROUS CURSOR_MAX_AGE_DAYS=10 (not the old 7-day warning + * threshold): the raised MAX_NEW_DECODES + the recent-first two-pass fetch + * (scripts/sync-bills.mjs) still need real nights to drain the pre-existing + * backlog once this change merges, and a threshold that insta-fails the + * very next run would block that catch-up window instead of giving it room + * to work. 10 days sits comfortably below lib/freshness-state.ts's + * FRESHNESS_DEAD_WINDOW_DAYS=21 (the site's own "this has gone genuinely + * dead" ceiling for the SAME cursor value), so CI catches a regression well + * before a visitor could ever see a dishonest "quiet week" from it. */ import { execSync } from 'node:child_process'; import { readFileSync } from 'node:fs'; +const CURSOR_MAX_AGE_DAYS = 10; + let failed = false; const fail = (msg) => { console.error(`::error::${msg}`); @@ -140,9 +157,11 @@ if (state) { ); } else { const cursorAgeDays = (Date.now() - Date.parse(state.lastSync)) / 86_400_000; - if (cursorAgeDays > 7) { - warn( - `corpus cursor is ${Math.round(cursorAgeDays)} days old (lastSync ${state.lastSync}). Expected while the decode backlog drains; if this warning persists after the backlog clears, promote it to a failure.` + // Hard failure, not a ::warning — see this file's header comment + // ("Cursor-age threshold") for why 10 days and why this was promoted. + if (cursorAgeDays > CURSOR_MAX_AGE_DAYS) { + fail( + `corpus cursor is ${Math.round(cursorAgeDays)} days old (lastSync ${state.lastSync}), past the ${CURSOR_MAX_AGE_DAYS}-day ceiling — the ascending backlog scan has stopped making real progress` ); } } diff --git a/tests/freshness.spec.ts b/tests/freshness.spec.ts index f421ef6..424a4e7 100644 --- a/tests/freshness.spec.ts +++ b/tests/freshness.spec.ts @@ -3,6 +3,7 @@ import syncState from '../data/sync-state.json'; import bills from '../data/bills.json'; import { TERMINAL_STATUSES, effectiveUrgency } from '../lib/urgency.mjs'; import { bandFloors } from '../lib/taxonomy'; +import { FRESHNESS_DEAD_WINDOW_DAYS, freshnessAgeDays } from '../lib/freshness-state'; /* * KTD-1 / KTD-2 / AE3. The stamp is baked at build time from getFreshness() @@ -41,6 +42,25 @@ const FRESH_CLOCK = LAST_RUN + 60 * 60 * 1000; // 1h after the last check const STALE_CLOCK = LAST_RUN + 10 * 86_400_000; // past the 5d claim window const DEAD_CLOCK = LAST_RUN + 30 * 86_400_000; // past the 21d dead window +// 2026-07-16 (audit §5 item 4): emptyStateVerdict no longer looks only at +// lastRun/checkedAt — the sync cursor (lastSync/completeThrough) and the +// corpus's own newest last_action_date now independently gate the verdict +// too (lib/freshness-state.ts). Mirror that here, corpus-derived exactly +// like anyNow/anyTop above, rather than hardcoding today's specific data — +// so these tests keep tracking the site's real behavior as the nightly sync +// rewrites data/ instead of silently drifting from it. +const newestActionDate = (bills as CorpusBill[]).reduce( + (max, b) => (b.last_action_date && b.last_action_date > max ? b.last_action_date : max), + '' +); +/** Whether the empty-state verdict reads data_stale AT FRESH_CLOCK for a + * reason that has nothing to do with lastRun (which is fresh by + * construction at FRESH_CLOCK) — i.e. the cursor or the corpus's newest + * known activity has gone dark past the dead window. */ +const contentStaleAtFreshClock = + freshnessAgeDays(syncState.lastSync, FRESH_CLOCK) > FRESHNESS_DEAD_WINDOW_DAYS || + freshnessAgeDays(newestActionDate, FRESH_CLOCK) > FRESHNESS_DEAD_WINDOW_DAYS; + /** Collect hydration-related console errors — the AE3 client verdict must * never be bought at the price of a server/client HTML mismatch. */ function trackHydrationErrors(page: Page): string[] { @@ -76,19 +96,29 @@ test.describe('freshness stamp reads from sync-state via the shared accessor', ( }); test.describe('AE3: quiet-week vs data-stale tri-state (homepage)', () => { - test('fresh clock: quiet week reads as quiet — and only on a truly quiet corpus', async ({ page }) => { + test('fresh clock: quiet week reads as quiet — and only on a truly quiet, genuinely current corpus', async ({ page }) => { const hydrationErrors = trackHydrationErrors(page); await page.clock.setFixedTime(FRESH_CLOCK); await page.goto('/'); const quietCard = page.getByRole('status').filter({ hasText: /Quiet week/ }); + const staleCard = page.getByRole('status').filter({ hasText: /Data check needed/ }); if (anyTop) { // Hot corpus: cards render, no quiet-week claim anywhere. await expect( page.locator('section[aria-labelledby="top-actions"] a[href*="/bills/"]').first() ).toBeVisible(); await expect(quietCard).toHaveCount(0); + } else if (contentStaleAtFreshClock) { + // The band is empty, but the sync cursor or the corpus's own newest + // activity is dead-window-stale — never claim "quiet" over that, even + // though lastRun (checkedAt) itself is fresh at this clock (2026-07-16 + // fix, audit §5 item 4: emptyStateVerdict no longer looks at lastRun + // alone). + await expect(staleCard).toBeVisible(); + await expect(quietCard).toHaveCount(0); } else if (!anyNow) { - // Genuinely quiet: the honest empty state, never a padded card. + // Genuinely quiet AND genuinely current: the honest empty state, never + // a padded card. await expect(quietCard).toBeVisible(); } else { // Floor cleared only by undecoded bills: no cards, but also no false @@ -102,7 +132,8 @@ test.describe('AE3: quiet-week vs data-stale tri-state (homepage)', () => { test.skip(anyNow, 'corpus not quiet this week — ES quiet-week copy not renderable'); await page.clock.setFixedTime(FRESH_CLOCK); await page.goto('/es'); - await expect(page.getByRole('status').filter({ hasText: 'Semana tranquila' })).toBeVisible(); + const text = contentStaleAtFreshClock ? 'Verificación pendiente' : 'Semana tranquila'; + await expect(page.getByRole('status').filter({ hasText: text })).toBeVisible(); }); test('stale clock: the empty slot says "data check needed", never "quiet"', async ({ page }) => { @@ -130,10 +161,18 @@ test.describe('AE3: /bills "Act now" band mirrors the same tri-state', () => { await page.clock.setFixedTime(FRESH_CLOCK); await page.goto('/bills'); const quietCard = page.getByRole('status').filter({ hasText: /Quiet week/ }); + const staleCard = page.getByRole('status').filter({ hasText: /Data check needed/ }); if (!anyNow) { - // The unfiltered now band renders the quiet-week card under its header. + // The unfiltered now band renders the empty-state card under its + // header — quiet_week only when the cursor/corpus are also genuinely + // current at this clock (audit §5 item 4), data_stale otherwise. await expect(page.locator('section[aria-labelledby="band-now"]').getByRole('status')).toBeVisible(); - await expect(quietCard).toBeVisible(); + if (contentStaleAtFreshClock) { + await expect(staleCard).toBeVisible(); + await expect(quietCard).toHaveCount(0); + } else { + await expect(quietCard).toBeVisible(); + } } else { await expect(page.locator('section[aria-labelledby="band-now"] a[href*="/bills/"]').first()).toBeVisible(); await expect(quietCard).toHaveCount(0); diff --git a/tests/freshness.unit.spec.ts b/tests/freshness.unit.spec.ts index caecac1..d8562f2 100644 --- a/tests/freshness.unit.spec.ts +++ b/tests/freshness.unit.spec.ts @@ -5,6 +5,7 @@ import { emptyStateVerdict, freshnessAgeDays, freshnessState, + type FreshnessSignals, } from '../lib/freshness-state'; // One frozen clock for both the input timestamp and the function's `now`, @@ -13,6 +14,16 @@ import { const NOW = Date.now(); const daysAgo = (n: number) => new Date(NOW - n * 86_400_000).toISOString(); +// All three signals fresh by default; each test overrides only the one +// signal it's exercising, so a failure clearly pins down which signal caused +// the verdict to flip. +const signals = (overrides: Partial = {}): FreshnessSignals => ({ + checkedAt: daysAgo(0), + completeThrough: daysAgo(0), + newestAction: daysAgo(0), + ...overrides, +}); + test.describe('freshnessState (KTD-2 fresh/stale/dead tri-state)', () => { test('fresh at 0 days and right at the claim-window boundary', () => { expect(freshnessState(daysAgo(0), NOW)).toBe('fresh'); @@ -35,15 +46,54 @@ test.describe('freshnessState (KTD-2 fresh/stale/dead tri-state)', () => { }); }); +// 2026-07-16 (audit §5 item 4): emptyStateVerdict's signature changed here, +// from a single `checkedAt` string to a FreshnessSignals object. Before this +// fix the verdict looked ONLY at checkedAt (lastRun) — "did the job run +// tonight" — so a pipeline that ran and committed every night but made no +// real forward progress (a frozen sync cursor, or a corpus with nothing +// genuinely new for weeks) still read as "fresh" and the site confidently +// claimed "quiet week" over data that was actually a month stale. That's the +// exact bug the audit found live: lastSync AND the corpus's newest +// last_action_date were both 29 days old while lastRun was only 2 days old. +// completeThrough and newestAction now independently gate the verdict too — +// see emptyStateVerdict's own doc comment for why they use the wider dead +// window rather than the tight claim window checkedAt uses. test.describe('emptyStateVerdict (the AE3 collapse rule)', () => { - test('fresh data + empty band = quiet_week', () => { - expect(emptyStateVerdict(daysAgo(0), NOW)).toBe('quiet_week'); - expect(emptyStateVerdict(daysAgo(FRESHNESS_CLAIM_WINDOW_DAYS), NOW)).toBe('quiet_week'); + test('every signal fresh + empty band = quiet_week', () => { + expect(emptyStateVerdict(signals(), NOW)).toBe('quiet_week'); + expect(emptyStateVerdict(signals({ checkedAt: daysAgo(FRESHNESS_CLAIM_WINDOW_DAYS) }), NOW)).toBe('quiet_week'); + }); + + test("checkedAt 'stale' or 'dead' collapses to data_stale — never quiet when the job itself hasn't run recently", () => { + expect(emptyStateVerdict(signals({ checkedAt: daysAgo(FRESHNESS_CLAIM_WINDOW_DAYS + 1) }), NOW)).toBe('data_stale'); + expect(emptyStateVerdict(signals({ checkedAt: daysAgo(FRESHNESS_DEAD_WINDOW_DAYS + 1) }), NOW)).toBe('data_stale'); + expect(emptyStateVerdict(signals({ checkedAt: 'not-a-date' }), NOW)).toBe('data_stale'); + }); + + test('a dead-window-stale sync cursor overrides a fresh checkedAt — never quiet on a frozen cursor', () => { + expect( + emptyStateVerdict(signals({ completeThrough: daysAgo(FRESHNESS_DEAD_WINDOW_DAYS + 1) }), NOW) + ).toBe('data_stale'); + }); + + test('a dead-window-stale newestAction overrides a fresh checkedAt — never quiet when nothing in the corpus is current', () => { + expect( + emptyStateVerdict(signals({ newestAction: daysAgo(FRESHNESS_DEAD_WINDOW_DAYS + 1) }), NOW) + ).toBe('data_stale'); }); - test("both 'stale' and 'dead' collapse to data_stale — never quiet on dead data", () => { - expect(emptyStateVerdict(daysAgo(FRESHNESS_CLAIM_WINDOW_DAYS + 1), NOW)).toBe('data_stale'); - expect(emptyStateVerdict(daysAgo(FRESHNESS_DEAD_WINDOW_DAYS + 1), NOW)).toBe('data_stale'); - expect(emptyStateVerdict('not-a-date', NOW)).toBe('data_stale'); + test('completeThrough/newestAction may lag WITHIN the dead window without tripping data_stale', () => { + // The sync cursor legitimately trails checkedAt by real days while the + // ascending backlog scan drains (lib/freshness.ts's own doc comment, + // scripts/sync-bills.mjs's two-pass fetch design note) — gating this + // signal on the tight claim window would make the site cry "data stale" + // most nights even when the recent-first pass kept things genuinely + // current. Right at the dead-window boundary is still quiet_week. + expect( + emptyStateVerdict( + signals({ completeThrough: daysAgo(FRESHNESS_DEAD_WINDOW_DAYS), newestAction: daysAgo(FRESHNESS_DEAD_WINDOW_DAYS) }), + NOW + ) + ).toBe('quiet_week'); }); }); diff --git a/tests/mcp-tools.spec.ts b/tests/mcp-tools.spec.ts index 647ba7b..9ac1515 100644 --- a/tests/mcp-tools.spec.ts +++ b/tests/mcp-tools.spec.ts @@ -1,5 +1,8 @@ import { expect, test } from '@playwright/test'; import { SITE_ORIGIN } from '../lib/site'; +import syncState from '../data/sync-state.json'; +import bills from '../data/bills.json'; +import { FRESHNESS_DEAD_WINDOW_DAYS, freshnessAgeDays, freshnessState } from '../lib/freshness-state'; import { callTool } from './helpers'; /* @@ -219,17 +222,40 @@ test.describe('search_bills', () => { }); }); +// This corpus's freshest last_action_date trails "today" by enough that no +// bill currently clears the S3 absolute urgency floor (pinned independently +// against the real data at authoring time) - the real, undoctored honesty +// case the spec calls for, not a mock. `data.bills` is empty either way; +// which of quiet_week/data_stale that empty result carries is corpus-derived +// below (mirroring lib/freshness-state.ts's emptyStateVerdict exactly) +// rather than hardcoded, so this test keeps tracking the site's real +// behavior as the nightly sync rewrites data/ instead of silently drifting +// from it. +// +// 2026-07-16 (audit §5 item 4): this invariant legitimately changed. +// emptyStateVerdict used to look ONLY at checkedAt/lastRun ("did the job +// run"), so this test could hardcode quiet_week=true/data_stale=false once +// and trust it. Now the sync cursor and the corpus's own newest activity +// independently gate the verdict too, and both are real fields the pipeline +// can leave stale even on nights lastRun itself looks fresh (the exact bug +// the audit found) - so the expectation has to be computed from the same +// three signals, not assumed. +const newestActionDate = (bills as { last_action_date: string | null }[]).reduce( + (max, b) => (b.last_action_date && b.last_action_date > max ? b.last_action_date : max), + '' +); +const expectDataStale = + freshnessState(syncState.lastRun) !== 'fresh' || + freshnessAgeDays(syncState.lastSync) > FRESHNESS_DEAD_WINDOW_DAYS || + freshnessAgeDays(newestActionDate) > FRESHNESS_DEAD_WINDOW_DAYS; + test.describe('whats_moving', () => { - test('honest empty during a genuine quiet week - never padded', async ({ request }) => { - // This corpus's freshest last_action_date trails "today" by enough that - // no bill currently clears the S3 absolute urgency floor (pinned - // independently against the real data at authoring time) - the real, - // undoctored honesty case the spec calls for, not a mock. + test('honest empty state matches the real corpus\'s freshness signals - never padded, never falsely quiet', async ({ request }) => { const result = await callTool(request, 'whats_moving', { locale: 'en' }); const data = result.structuredContent!; expect(data.bills).toEqual([]); - expect(data.quiet_week).toBe(true); - expect(data.data_stale).toBe(false); // fresh pipeline, genuinely quiet - not a stale-data false alarm + expect(data.quiet_week).toBe(!expectDataStale); + expect(data.data_stale).toBe(expectDataStale); expect(data.days).toBe(7); // An empty, non-AI result carries no AI label - nothing to disclose. expectMeta(data.meta as Record, '/', false);