diff --git a/.github/workflows/daily-metrics.yml b/.github/workflows/daily-metrics.yml
index 279f178..3ff51f0 100644
--- a/.github/workflows/daily-metrics.yml
+++ b/.github/workflows/daily-metrics.yml
@@ -70,4 +70,14 @@ jobs:
UPSTASH_COUNTERS_REST_TOKEN: ${{ secrets.UPSTASH_COUNTERS_REST_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
DIGEST_ISSUE_NUMBER: ${{ steps.setup.outputs.issue_num }}
+ # Flagged (not fixed) in PR #87: this workflow never set VERCEL_ENV,
+ # so lib/upstash.ts's keyPrefix() resolved to 'dev' here while the
+ # live site's routes write under 'production:' (VERCEL_ENV is
+ # Vercel-injected at request time, not something a GitHub Actions
+ # runner has). Every readUsageWindow/readMcpClientDay call was
+ # silently reading an empty 'dev:' keyspace instead of the real
+ # counters - the digest could only ever report zeros. Same fix
+ # scripts/tenant-admin.mjs's own header comment already documents
+ # for laptop-shell CLI runs: export VERCEL_ENV=production explicitly.
+ VERCEL_ENV: production
run: npx tsx scripts/daily-metrics.mjs
diff --git a/.github/workflows/hot-bills.yml b/.github/workflows/hot-bills.yml
new file mode 100644
index 0000000..2ea111f
--- /dev/null
+++ b/.github/workflows/hot-bills.yml
@@ -0,0 +1,89 @@
+name: Hot-bill refresh
+
+# Twice-daily, refresh-ONLY pass between nightly bill syncs (audit §4 Alt B /
+# §5 item 3, pipeline-audit.md): the nightly sync (sync-bills.yml, 07:30 UTC)
+# structurally can't reflect a floor vote or markup that happens mid-day
+# until the next morning. This job re-fetches the ~100 most-recently-updated
+# Congress.gov bills and refreshes status/last_action_date/urgency for
+# whichever of them are ALREADY in the corpus - decodes nothing, so a
+# brand-new bill still waits for the nightly sync's decode-before-publish
+# gate (scripts/hot-bills.mjs's header comment has the full reasoning).
+# Zero Anthropic usage: no ANTHROPIC_API_KEY in this workflow at all.
+
+on:
+ schedule:
+ # 17:00 and 22:00 UTC (~1pm/6pm ET) - both inside the US legislative day,
+ # after floor/committee activity has had time to post to Congress.gov,
+ # and well clear of the 07:30 UTC nightly sync on both sides.
+ - cron: '0 17 * * *'
+ - cron: '0 22 * * *'
+ workflow_dispatch:
+
+permissions:
+ contents: write
+
+concurrency:
+ # Shared with sync-bills.yml/refresh-legislators.yml so this can never race
+ # the nightly sync's commit - same reasoning as those two workflows already
+ # sharing this group.
+ group: data-sync
+ cancel-in-progress: false
+
+jobs:
+ hot-refresh:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v7
+ - uses: actions/setup-node@v6
+ # scripts/hot-bills.mjs and scripts/congress-fetch.mjs are stdlib-only
+ # (Node's global fetch + lib/urgency.mjs, a plain .mjs with no npm
+ # deps) - same "the runner's node suffices, no npm ci" posture as
+ # refresh-legislators.yml and scripts/verify-deploy.mjs.
+ with:
+ node-version: 24
+ - name: Refresh hot bills
+ env:
+ CONGRESS_API_KEY: ${{ secrets.CONGRESS_API_KEY }}
+ run: node scripts/hot-bills.mjs
+ - name: Commit data
+ id: commit
+ # Same oravan-sync author pattern as sync-bills.yml/refresh-
+ # legislators.yml - the author email must map to the Vercel-linked
+ # GitHub account or Vercel BLOCKs the auto-deploy (docs/solutions/
+ # vercel-bot-push-blocked-deploys.md).
+ run: |
+ git config user.name "oravan-sync"
+ git config user.email "223600121+cm2489@users.noreply.github.com"
+ git add data/
+ if git diff --cached --quiet; then
+ echo "no changes"
+ echo "changed=false" >> "$GITHUB_OUTPUT"
+ exit 0
+ fi
+ echo "changed=true" >> "$GITHUB_OUTPUT"
+ git commit -m "chore(data): hot-bill refresh $(date -u +%FT%HZ)"
+ # This job only ever touches data/bills.json (refresh-only, no ES/
+ # coverage/sync-state writes) - disjoint from sync-bills.yml's and
+ # refresh-legislators.yml's own file sets, but the checked-out base
+ # can still go stale behind either of them in the shared data-sync
+ # concurrency group. Rebase onto the latest main and retry, same
+ # pattern as the other two workflows.
+ for i in 1 2 3 4 5; do
+ if git push; then
+ echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
+ exit 0
+ fi
+ echo "push rejected, rebasing onto latest main (attempt $i)"
+ git fetch origin main
+ git rebase origin/main
+ done
+ exit 1
+ - name: Verify the deploy landed
+ # Same dead-man's-switch as sync-bills.yml/refresh-legislators.yml -
+ # poll production until it serves the SHA we just pushed. Skips with
+ # a notice until the PROD_URL repo variable is set.
+ if: steps.commit.outputs.changed == 'true'
+ env:
+ PROD_URL: ${{ vars.PROD_URL }}
+ EXPECT_SHA: ${{ steps.commit.outputs.sha }}
+ run: node scripts/verify-deploy.mjs
diff --git a/.github/workflows/sync-bills.yml b/.github/workflows/sync-bills.yml
index e3d0d11..6d96b3d 100644
--- a/.github/workflows/sync-bills.yml
+++ b/.github/workflows/sync-bills.yml
@@ -7,7 +7,7 @@ on:
inputs:
max_new_decodes:
description: 'Max new bills to AI-decode this run'
- default: '40'
+ default: '120'
permissions:
contents: write
@@ -35,7 +35,7 @@ jobs:
env:
CONGRESS_API_KEY: ${{ secrets.CONGRESS_API_KEY }}
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
- MAX_NEW_DECODES: ${{ inputs.max_new_decodes || '40' }}
+ MAX_NEW_DECODES: ${{ inputs.max_new_decodes || '120' }}
run: node scripts/sync-bills.mjs
- name: Backfill coverage search inputs
# Drains press_names/news_query for bills decoded before search-input
@@ -143,6 +143,15 @@ jobs:
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
UPSTASH_CACHE_REST_URL: ${{ secrets.UPSTASH_CACHE_REST_URL }}
UPSTASH_CACHE_REST_TOKEN: ${{ secrets.UPSTASH_CACHE_REST_TOKEN }}
+ # 2026-07-16 metrics-env fix (same root cause as PR #87's "Flagged,
+ # not fixed" note on daily-metrics.yml): this workflow never set
+ # VERCEL_ENV either, so lib/scriptcache.ts's keys were written under
+ # keyPrefix()'s 'dev' fallback instead of 'production' — every combo
+ # this step ever cached since PREGEN_ENABLED was armed (2026-07-12)
+ # landed in a keyspace app/api/script/route.ts's live requests
+ # (VERCEL_ENV=production, Vercel-injected) never read, so every
+ # pregen run has been pure spend with zero possible cache hits.
+ VERCEL_ENV: production
run: |
if [ "$PREGEN_ENABLED" != "true" ]; then
echo "::notice::pregen SKIPPED - repo variable PREGEN_ENABLED is not 'true'. \$0 spent tonight. See the S21 PR's Arming checklist to turn this on."
diff --git a/app/[locale]/bills/page.tsx b/app/[locale]/bills/page.tsx
index b2e42e8..db58096 100644
--- a/app/[locale]/bills/page.tsx
+++ b/app/[locale]/bills/page.tsx
@@ -37,7 +37,7 @@ export default async function BillsPage({ params }: { params: Promise<{ locale:
{t(`bills.bandSub.${band}`)}
diff --git a/lib/core/mcp.ts b/lib/core/mcp.ts
index 1134299..82d7422 100644
--- a/lib/core/mcp.ts
+++ b/lib/core/mcp.ts
@@ -565,11 +565,14 @@ export function whatsMoving(params: WhatsMovingParams, locale: Locale) {
* AE3/KTD-2 honesty rule, reusing lib/freshness-state.ts's collapse rather
* than re-deriving it (that file's own doc comment names this exact
* tool): an empty result reads as a genuine "quiet week" only while the
- * nightly pipeline itself looks alive. A stale or dead pipeline must never
- * be dressed up as "nothing to act on this week" - that would hand an
- * agent a fact about our sync health disguised as a fact about Congress.
+ * nightly pipeline itself looks alive AND the sync cursor/corpus itself
+ * shows real recent progress (not just "the job executed" - see
+ * emptyStateVerdict's own doc comment for why both signals matter). A
+ * stale or dead pipeline must never be dressed up as "nothing to act on
+ * this week" - that would hand an agent a fact about our sync health
+ * disguised as a fact about Congress.
*/
- const verdict = limited.length === 0 ? emptyStateVerdict(getFreshness().checkedAt) : null;
+ const verdict = limited.length === 0 ? emptyStateVerdict(getFreshness()) : null;
return {
bills: limited,
diff --git a/lib/freshness-state.ts b/lib/freshness-state.ts
index 9d3531c..4ba1ae6 100644
--- a/lib/freshness-state.ts
+++ b/lib/freshness-state.ts
@@ -40,16 +40,56 @@ export function freshnessState(checkedAt: string, now: number = Date.now()): Fre
export type EmptyStateVerdict = 'quiet_week' | 'data_stale';
+/** The three freshness signals `emptyStateVerdict` reads — structurally the
+ * same shape as lib/freshness.ts's `Freshness`, redeclared here (rather than
+ * imported) so this file stays free of any data import and safe to ship to
+ * a client bundle, per the header comment above. */
+export interface FreshnessSignals {
+ /** Last successful nightly sync run — "did the job run at all". */
+ checkedAt: string;
+ /** Sync cursor high-water mark — "how far the backlog scan has actually
+ * processed" (data/sync-state.json's lastSync). */
+ completeThrough: string;
+ /** Newest `last_action_date` across the whole corpus — "is there anything
+ * current in the data at all", regardless of what the sync's own
+ * bookkeeping claims. */
+ newestAction: string;
+}
+
/**
* The AE3 collapse rule as an importable primitive: an empty "Act now" band
- * is a quiet week only while the data is fresh; both 'stale' and 'dead'
- * collapse to data_stale — never assert quiet on dead data. This is the ONE
- * copy of that rule: the site's empty band renders it (components/
- * UrgencyEmptyState.tsx), and the future MCP `whats_moving` tool must import
- * this same function rather than re-deriving the collapse inline — a second
- * copy is the exact drift docs/solutions/stale-urgency-freeze.md closed for
- * the urgency curve.
+ * reads as a genuine quiet_week only when EVERY freshness signal checks out.
+ * This is the ONE copy of that rule: the site's empty band renders it
+ * (components/UrgencyEmptyState.tsx), and lib/core/mcp.ts's `whatsMoving`
+ * imports this same function rather than re-deriving the collapse inline —
+ * a second copy is the exact drift docs/solutions/stale-urgency-freeze.md
+ * closed for the urgency curve.
+ *
+ * Two different thresholds, deliberately (2026-07-16, audit §5 item 4):
+ * - `checkedAt` (did the nightly job even run tonight) uses the tight
+ * FRESHNESS_CLAIM_WINDOW_DAYS/FRESHNESS_DEAD_WINDOW_DAYS pair via
+ * freshnessState — a "we checked recently" claim should go stale fast.
+ * - `completeThrough` (the sync cursor) and `newestAction` (the corpus's
+ * own newest activity) instead trip data_stale only past
+ * FRESHNESS_DEAD_WINDOW_DAYS, the wider of the two constants. Both are
+ * EXPECTED to lag `checkedAt` by real days under ordinary operation — the
+ * ascending backlog-scan cursor deliberately trails while it drains (see
+ * lib/freshness.ts's own doc comment and scripts/sync-bills.mjs's
+ * two-pass fetch design note) — so gating them on the tight claim window
+ * would make the site cry "data stale" every single night even when
+ * tonight's recent-first pass kept the actually-relevant content current.
+ * The wide dead window instead catches the failure mode this item exists
+ * for: a pipeline that runs every night, commits every night, and reports
+ * itself "fresh" via `checkedAt` alone while making no real forward
+ * progress for weeks (the bug this audit found — a 29-day-old cursor and
+ * a 29-day-old newest action, both silently passing as "fresh" under the
+ * old checkedAt-only check). Past three weeks with NOTHING new anywhere
+ * in the corpus, "quiet week" stops being a credible claim regardless of
+ * how recently the job merely executed.
*/
-export function emptyStateVerdict(checkedAt: string, now: number = Date.now()): EmptyStateVerdict {
- return freshnessState(checkedAt, now) === 'fresh' ? 'quiet_week' : 'data_stale';
+export function emptyStateVerdict(signals: FreshnessSignals, now: number = Date.now()): EmptyStateVerdict {
+ if (freshnessState(signals.checkedAt, now) !== 'fresh') return 'data_stale';
+ if (freshnessAgeDays(signals.completeThrough, now) > FRESHNESS_DEAD_WINDOW_DAYS) return 'data_stale';
+ if (freshnessAgeDays(signals.newestAction, now) > FRESHNESS_DEAD_WINDOW_DAYS) return 'data_stale';
+ return 'quiet_week';
}
diff --git a/scripts/congress-fetch.mjs b/scripts/congress-fetch.mjs
new file mode 100644
index 0000000..62d9afb
--- /dev/null
+++ b/scripts/congress-fetch.mjs
@@ -0,0 +1,152 @@
+/**
+ * Congress.gov fetch + status/urgency/category-mapping helpers shared by the
+ * nightly bill sync (scripts/sync-bills.mjs: fetch + AI decode + commit) and
+ * the twice-daily hot-bill refresh (scripts/hot-bills.mjs: fetch + refresh
+ * only, zero AI cost) - extracted 2026-07-16 (audit §4 Alt B / §5 item 3) so
+ * the two scripts share one implementation of "talk to Congress.gov" and
+ * "map a bill-detail payload onto our fields" instead of maintaining two
+ * copies that can drift (the same "one copy" discipline lib/urgency.mjs's
+ * own doc comment already applies to the urgency curve).
+ *
+ * Needs CONGRESS_API_KEY in the importing process's env.
+ */
+import { STATUS_BASE } from '../lib/urgency.mjs';
+
+export const CONGRESS = 119;
+export const BILL_TYPES = new Set(['hr', 's', 'hjres', 'sjres']);
+
+const API = 'https://api.congress.gov/v3';
+const KEY = process.env.CONGRESS_API_KEY;
+if (!KEY) throw new Error('CONGRESS_API_KEY missing');
+
+/** GET one Congress.gov endpoint, retrying on a bad status or a thrown/timed
+ * out request (a hung socket must retry, not kill the whole run - the
+ * 2026-06-13 crash). */
+export async function cg(path, params = {}) {
+ const url = new URL(`${API}${path}`);
+ for (const [k, v] of Object.entries(params)) url.searchParams.set(k, v);
+ url.searchParams.set('api_key', KEY);
+ url.searchParams.set('format', 'json');
+ let lastErr;
+ for (let attempt = 0; attempt <= 4; attempt++) {
+ if (attempt > 0) await new Promise((r) => setTimeout(r, 2000 * attempt));
+ try {
+ // 30s per-request ceiling: a hung socket fails fast and retries instead
+ // of hanging on undici's ~5min headers timeout (the 2026-06-13 crash).
+ const res = await fetch(url, { signal: AbortSignal.timeout(30_000) });
+ // await inside the try: the 30s abort can fire mid-body-read, and an
+ // un-awaited res.json() rejection would escape the catch and kill the
+ // run uncaught instead of retrying (the 2026-07-04 crash).
+ if (res.ok) return await res.json();
+ lastErr = new Error(`Congress.gov ${res.status} for ${path}`);
+ } catch (e) {
+ lastErr = e; // network error / timeout - retry rather than kill the run
+ }
+ }
+ throw lastErr;
+}
+
+/** The N most-recently-updated bills across the whole corpus (no fromDateTime
+ * floor - literally "what changed most recently"), of our 4 tracked types.
+ * Used by sync-bills.mjs's recent-first pass and the whole of hot-bills.mjs;
+ * see the audit's two-pass fetch design (§5 item 2 / §4 Alt B): the
+ * ascending "since cursor" scan structurally reaches the newest bills LAST,
+ * so both freshness-sensitive callers fetch this descending window instead. */
+export async function fetchRecentlyUpdated(limit) {
+ const page = await cg(`/bill/${CONGRESS}`, { sort: 'updateDate+desc', limit });
+ const items = page.bills ?? [];
+ return items.filter((b) => BILL_TYPES.has((b.type ?? '').toLowerCase()));
+}
+
+// ---- status mapping (ported from the reference implementation) ----
+export function mapStatus(actionText) {
+ const text = (actionText ?? '').toLowerCase().trim();
+ if (!text) return 'committee';
+ if (text.includes('became public law') || text.includes('signed by president')) return 'signed';
+ if (text.includes('vetoed')) return 'vetoed';
+ if (text.includes('conference report') || text.includes('conference committee')) return 'conference';
+ if (
+ text.includes('passed house') || text.includes('passed senate') ||
+ text.includes('passed/agreed to') || text.includes('agreed to in') ||
+ text.includes('received in the senate') || text.includes('received in the house') ||
+ text.includes('held at the desk')
+ ) return 'passed_chamber';
+ if (
+ text.includes('placed on') || text.includes('calendar') ||
+ text.includes('cloture') || text.includes('rule provid') ||
+ text.includes('motion to proceed')
+ ) return 'floor_vote';
+ if (text.includes('markup') || text.includes('ordered to be reported') || text.includes('reported by')) return 'markup';
+ return 'committee';
+}
+
+// Stored sync-time score (freshness bonus, no decay) - the FEED never ranks
+// by this; read-time effectiveUrgency in lib/urgency.mjs does the ranking.
+// The base table is shared so the two curves can't drift apart.
+export function urgencyScore(status, lastActionDate) {
+ const base = STATUS_BASE[status] ?? 0.2;
+ let bonus = 0;
+ if (lastActionDate) {
+ const days = (Date.now() - new Date(lastActionDate).getTime()) / 86_400_000;
+ if (Number.isFinite(days)) bonus = days < 3 ? 0.1 : days < 7 ? 0.05 : 0;
+ }
+ return Math.round(Math.min(1, Math.max(0, base + bonus)) * 1000) / 1000;
+}
+
+// CRS Policy Area -> our 12 flat categories (1:1, all 32 areas covered)
+const POLICY_AREA_TO_CATEGORY = {
+ 'Labor and Employment': 'jobs_economy', 'Commerce': 'jobs_economy',
+ 'Finance and Financial Sector': 'jobs_economy', 'Taxation': 'jobs_economy',
+ 'Economics and Public Finance': 'jobs_economy', 'Agriculture and Food': 'jobs_economy',
+ 'Transportation and Public Works': 'jobs_economy',
+ 'Science, Technology, Communications': 'ai_technology',
+ 'Health': 'health',
+ 'Housing and Community Development': 'housing',
+ 'Immigration': 'immigration',
+ 'Government Operations and Politics': 'government_democracy', 'Congress': 'government_democracy',
+ 'Emergency Management': 'government_democracy',
+ 'Crime and Law Enforcement': 'crime_justice', 'Law': 'crime_justice',
+ 'Education': 'education', 'Sports and Recreation': 'education',
+ 'Social Sciences and History': 'education',
+ 'Environmental Protection': 'environment_energy', 'Energy': 'environment_energy',
+ 'Public Lands and Natural Resources': 'environment_energy',
+ 'Water Resources Development': 'environment_energy', 'Animals': 'environment_energy',
+ 'Civil Rights and Liberties, Minority Issues': 'rights_liberties',
+ 'Armed Forces and National Security': 'national_security',
+ 'International Affairs': 'national_security',
+ 'Foreign Trade and International Finance': 'national_security',
+ 'Families': 'family_community', 'Social Welfare': 'family_community',
+ 'Native Americans': 'family_community', 'Arts, Culture, Religion': 'family_community',
+};
+
+export function tagBill(policyArea) {
+ const cat = POLICY_AREA_TO_CATEGORY[policyArea ?? ''];
+ return cat ? [cat] : [];
+}
+
+export function slugOf(b) {
+ return `${b.bill_type}-${b.bill_number}-${b.congress_number}`.toLowerCase();
+}
+
+/** Slug for a Congress.gov bill-list item ({type, number}), not yet a corpus
+ * bill object - the shape sync-bills.mjs's `updated`/recent-pass arrays and
+ * hot-bills.mjs's fetch results are in. */
+export function updateSlug(u, congress = CONGRESS) {
+ return `${u.type.toLowerCase()}-${u.number}-${congress}`.toLowerCase();
+}
+
+/** Mutate an existing corpus bill's refreshable fields in place from a
+ * Congress.gov bill-detail payload (`cg('/bill/{congress}/{type}/{number}')`'s
+ * `.bill`). Free, no AI cost - the one place both scripts' "refresh" branch
+ * lives, so it can't drift between the nightly sync and the hot-bill pass. */
+export function refreshBillFields(existing, detail) {
+ const status = mapStatus(detail.latestAction?.text);
+ const lastActionDate = detail.latestAction?.actionDate ?? null;
+ existing.status = status;
+ existing.last_action_date = lastActionDate;
+ existing.last_action_text = detail.latestAction?.text ?? existing.last_action_text;
+ existing.urgency_score = urgencyScore(status, lastActionDate);
+ const tags = tagBill(detail.policyArea?.name);
+ if (tags.length) existing.issue_tags = tags;
+ existing.policy_area = detail.policyArea?.name ?? existing.policy_area;
+}
diff --git a/scripts/hot-bills.mjs b/scripts/hot-bills.mjs
new file mode 100644
index 0000000..fab6864
--- /dev/null
+++ b/scripts/hot-bills.mjs
@@ -0,0 +1,66 @@
+/**
+ * Twice-daily hot-bill refresh (audit §4 Alt B / §5 item 3). REFRESH-ONLY:
+ * updates status/last_action_date/last_action_text/urgency_score/issue_tags
+ * for bills ALREADY in data/bills.json, using the same "~100 most-recently-
+ * updated across the whole 119th Congress" Congress.gov window
+ * scripts/sync-bills.mjs's recent-first pass uses (scripts/congress-fetch.mjs,
+ * shared so the two scripts can't drift).
+ *
+ * node --env-file=.env.local scripts/hot-bills.mjs
+ *
+ * Decodes NOTHING. A brand-new bill discovered here is left for the nightly
+ * sync (scripts/sync-bills.mjs) - that script is the only place the
+ * decode-before-publish gate runs, so this script must never publish an
+ * undecoded bill. Zero Anthropic usage: needs only CONGRESS_API_KEY. This is
+ * the tradeoff named up front in the audit's Alt B - a bill that's brand new
+ * AND breaking mid-day still waits until the next 07:30 UTC nightly sync to
+ * actually appear on the site; only bills already in the corpus get same-day
+ * status/urgency freshness from this pass.
+ *
+ * Runs 2x/day (.github/workflows/hot-bills.yml, 17:00 + 22:00 UTC - inside
+ * the US legislative day) between nightly syncs, so a floor vote or markup
+ * that happens mid-day is reflected in effectiveUrgency (lib/urgency.mjs)
+ * same-day instead of sitting stale until the next morning's sync. Also
+ * directly improves lib/freshness.ts's `newestAction` signal (scanned live
+ * from data/bills.json's last_action_date values), independent of
+ * data/sync-state.json - this script intentionally never touches
+ * sync-state.json; that file's lastRun/lastSync are the NIGHTLY sync's own
+ * "did the job run" / "how far has the backlog scan processed" signals, and
+ * conflating a same-day refresh pass with the sync cursor would work against
+ * lib/freshness-state.ts's honesty model, not for it.
+ */
+import { readFileSync, writeFileSync } from 'node:fs';
+import { CONGRESS, cg, fetchRecentlyUpdated, refreshBillFields, slugOf, updateSlug } from './congress-fetch.mjs';
+
+const FETCH_LIMIT = Number(process.env.HOT_BILLS_FETCH_LIMIT ?? 100);
+
+const bills = JSON.parse(readFileSync('data/bills.json', 'utf8'));
+const bySlug = new Map(bills.map((b) => [slugOf(b), b]));
+
+console.log(`hot-bill refresh: fetching up to ${FETCH_LIMIT} most-recently-updated bills`);
+const recent = await fetchRecentlyUpdated(FETCH_LIMIT);
+
+let refreshed = 0, newSkipped = 0, failed = 0;
+for (const u of recent) {
+ const type = u.type.toLowerCase();
+ const slug = updateSlug(u);
+ const existing = bySlug.get(slug);
+ if (!existing) {
+ newSkipped++; // brand-new bill - decode-before-publish waits for the nightly sync
+ continue;
+ }
+ try {
+ const { bill: d } = await cg(`/bill/${CONGRESS}/${type}/${u.number}`);
+ refreshBillFields(existing, d);
+ refreshed++;
+ } catch (e) {
+ failed++;
+ console.error(`FAIL ${slug}: ${e.message}`);
+ }
+}
+
+writeFileSync('data/bills.json', JSON.stringify(bills));
+console.log(
+ `DONE: ${refreshed} refreshed, ${newSkipped} new bill(s) skipped (nightly sync decodes those), ${failed} failed; corpus ${bills.length}`
+);
+if (failed > recent.length / 2) process.exit(1); // mostly-failed run: don't let CI commit garbage
diff --git a/scripts/sync-bills.mjs b/scripts/sync-bills.mjs
index d1c552e..8dd8c02 100644
--- a/scripts/sync-bills.mjs
+++ b/scripts/sync-bills.mjs
@@ -12,19 +12,69 @@
* their EN+ES summary and headline exist, so the feed never shows
* undecoded entries. At most MAX_NEW_DECODES per run (cost ceiling);
* the rest wait for the next night.
+ *
+ * Two-pass fetch (2026-07-16, audit §5 item 2). Congress.gov is queried
+ * TWICE per run, in this order:
+ * 1. Recent-first: `sort=updateDate+desc, limit=RECENT_FETCH_LIMIT` - the
+ * ~100 most-recently-touched bills in the whole 119th Congress, no
+ * cursor floor. Already-known bills refresh for free; brand-new bills
+ * decode within a RESERVED sub-budget (RECENT_DECODE_RESERVE, carved
+ * OUT of MAX_NEW_DECODES, not additional). This exists because the
+ * ascending backlog scan below structurally reaches the newest bills
+ * LAST - on a night with a deep backlog (or a busy legislative day) a
+ * floor vote that just happened would otherwise lose the race against
+ * both MAX_UPDATES and MAX_NEW_DECODES every single night, which is
+ * exactly how HR 7378 (and the whole "worth a call" feed) went stale
+ * for weeks even on clean, successful runs (see the audit).
+ * 2. Ascending backlog: `fromDateTime: lastSync, sort=updateDate+asc` -
+ * unchanged from before, drains the historical backlog oldest-first
+ * with whatever decode budget the recent-first pass didn't use. A bill
+ * already handled by pass 1 this run is skipped here (deduped, not
+ * re-fetched or re-decoded).
+ *
+ * CURSOR SEMANTICS (load-bearing, KTD-pinned): `state.lastSync`'s freeze-
+ * on-incomplete-work high-water mark is advanced ONLY by the ascending pass
+ * below. The recent-first pass never reads or writes `cursor`/`frozen` - it
+ * can find and decode a bill from last week while the ascending backlog is
+ * still stuck in May, and the cursor must keep meaning "the backlog scan has
+ * fully processed through here", not silently jump forward just because a
+ * recent bill happened to get handled out of order. See
+ * docs/solutions/pinned-sync-cursor.md for why an all-or-nothing cursor is
+ * exactly the failure this preserves the fix for.
*/
import Anthropic from '@anthropic-ai/sdk';
import { readFileSync, writeFileSync } from 'node:fs';
-import { STATUS_BASE } from '../lib/urgency.mjs';
+import {
+ BILL_TYPES,
+ CONGRESS,
+ cg,
+ fetchRecentlyUpdated,
+ mapStatus,
+ refreshBillFields,
+ slugOf,
+ tagBill,
+ updateSlug,
+ urgencyScore,
+} from './congress-fetch.mjs';
import { generateSearchInputs } from './search-inputs.mjs';
-const CONGRESS = 119;
-const BILL_TYPES = new Set(['hr', 's', 'hjres', 'sjres']);
const MAX_UPDATES = Number(process.env.MAX_UPDATES ?? 500);
-const MAX_NEW_DECODES = Number(process.env.MAX_NEW_DECODES ?? 40);
-const API = 'https://api.congress.gov/v3';
-const KEY = process.env.CONGRESS_API_KEY;
-if (!KEY) throw new Error('CONGRESS_API_KEY missing');
+// Raised 40 -> 120 (2026-07-16, audit §5 item 1): live nightly logs showed
+// 373-418 bills/night needing decode against a 40-bill budget, pinning the
+// ascending-pass cursor (state.lastSync) weeks behind and starving newer
+// bills of decode slots night after night. 120 doesn't fully clear that
+// inflow alone (~$8-14/night at $0.07-0.15/bill) - see the two-pass fetch
+// design note above for the fix that stops recency from losing the race
+// structurally, independent of how large the budget is.
+const MAX_NEW_DECODES = Number(process.env.MAX_NEW_DECODES ?? 120);
+// The recent-first pass's fetch window (audit §5 item 2 / §4 Alt A) - same
+// rough size as the twice-daily hot-bills.mjs refresh pass.
+const RECENT_FETCH_LIMIT = Number(process.env.RECENT_FETCH_LIMIT ?? 100);
+// New-bill decode budget RESERVED for the recent-first pass, carved out of
+// (not additional to) MAX_NEW_DECODES - a night with zero brand-new bills in
+// the last ~100 updates leaves the full MAX_NEW_DECODES for the ascending
+// backlog pass; a night with several leaves proportionally less.
+const RECENT_DECODE_RESERVE = Number(process.env.RECENT_DECODE_RESERVE ?? 20);
const anthropic = new Anthropic({ maxRetries: 8 });
// Sonnet 5's tokenizer runs ~30% more tokens than 4.6 for the same text, so
@@ -38,10 +88,6 @@ const es = JSON.parse(readFileSync('data/bills-es.json', 'utf8'));
const state = JSON.parse(readFileSync('data/sync-state.json', 'utf8'));
const bySlug = new Map(bills.map((b) => [slugOf(b), b]));
-function slugOf(b) {
- return `${b.bill_type}-${b.bill_number}-${b.congress_number}`.toLowerCase();
-}
-
// Congress.gov's bill-list `updateDate` field is date-only (e.g. "2026-06-04"),
// not a full timestamp. Persisting it as-is breaks the next run's fromDateTime
// query, which Congress.gov 400s on - the 2026-06-25/07-01 outage. Always
@@ -50,96 +96,6 @@ function toISODateTime(d) {
return /T/.test(d) ? d : `${d}T00:00:00Z`;
}
-async function cg(path, params = {}) {
- const url = new URL(`${API}${path}`);
- for (const [k, v] of Object.entries(params)) url.searchParams.set(k, v);
- url.searchParams.set('api_key', KEY);
- url.searchParams.set('format', 'json');
- let lastErr;
- for (let attempt = 0; attempt <= 4; attempt++) {
- if (attempt > 0) await new Promise((r) => setTimeout(r, 2000 * attempt));
- try {
- // 30s per-request ceiling: a hung socket fails fast and retries instead
- // of hanging on undici's ~5min headers timeout (the 2026-06-13 crash).
- const res = await fetch(url, { signal: AbortSignal.timeout(30_000) });
- // await inside the try: the 30s abort can fire mid-body-read, and an
- // un-awaited res.json() rejection would escape the catch and kill the
- // run uncaught instead of retrying (the 2026-07-04 crash).
- if (res.ok) return await res.json();
- lastErr = new Error(`Congress.gov ${res.status} for ${path}`);
- } catch (e) {
- lastErr = e; // network error / timeout - retry rather than kill the run
- }
- }
- throw lastErr;
-}
-
-// ---- status mapping (ported from the reference implementation) ----
-function mapStatus(actionText) {
- const text = (actionText ?? '').toLowerCase().trim();
- if (!text) return 'committee';
- if (text.includes('became public law') || text.includes('signed by president')) return 'signed';
- if (text.includes('vetoed')) return 'vetoed';
- if (text.includes('conference report') || text.includes('conference committee')) return 'conference';
- if (
- text.includes('passed house') || text.includes('passed senate') ||
- text.includes('passed/agreed to') || text.includes('agreed to in') ||
- text.includes('received in the senate') || text.includes('received in the house') ||
- text.includes('held at the desk')
- ) return 'passed_chamber';
- if (
- text.includes('placed on') || text.includes('calendar') ||
- text.includes('cloture') || text.includes('rule provid') ||
- text.includes('motion to proceed')
- ) return 'floor_vote';
- if (text.includes('markup') || text.includes('ordered to be reported') || text.includes('reported by')) return 'markup';
- return 'committee';
-}
-
-// Stored sync-time score (freshness bonus, no decay) - the FEED never ranks
-// by this; read-time effectiveUrgency in lib/urgency.mjs does the ranking.
-// The base table is shared so the two curves can't drift apart.
-function urgencyScore(status, lastActionDate) {
- const base = STATUS_BASE[status] ?? 0.2;
- let bonus = 0;
- if (lastActionDate) {
- const days = (Date.now() - new Date(lastActionDate).getTime()) / 86_400_000;
- if (Number.isFinite(days)) bonus = days < 3 ? 0.1 : days < 7 ? 0.05 : 0;
- }
- return Math.round(Math.min(1, Math.max(0, base + bonus)) * 1000) / 1000;
-}
-
-// CRS Policy Area -> our 12 flat categories (1:1, all 32 areas covered)
-const POLICY_AREA_TO_CATEGORY = {
- 'Labor and Employment': 'jobs_economy', 'Commerce': 'jobs_economy',
- 'Finance and Financial Sector': 'jobs_economy', 'Taxation': 'jobs_economy',
- 'Economics and Public Finance': 'jobs_economy', 'Agriculture and Food': 'jobs_economy',
- 'Transportation and Public Works': 'jobs_economy',
- 'Science, Technology, Communications': 'ai_technology',
- 'Health': 'health',
- 'Housing and Community Development': 'housing',
- 'Immigration': 'immigration',
- 'Government Operations and Politics': 'government_democracy', 'Congress': 'government_democracy',
- 'Emergency Management': 'government_democracy',
- 'Crime and Law Enforcement': 'crime_justice', 'Law': 'crime_justice',
- 'Education': 'education', 'Sports and Recreation': 'education',
- 'Social Sciences and History': 'education',
- 'Environmental Protection': 'environment_energy', 'Energy': 'environment_energy',
- 'Public Lands and Natural Resources': 'environment_energy',
- 'Water Resources Development': 'environment_energy', 'Animals': 'environment_energy',
- 'Civil Rights and Liberties, Minority Issues': 'rights_liberties',
- 'Armed Forces and National Security': 'national_security',
- 'International Affairs': 'national_security',
- 'Foreign Trade and International Finance': 'national_security',
- 'Families': 'family_community', 'Social Welfare': 'family_community',
- 'Native Americans': 'family_community', 'Arts, Culture, Religion': 'family_community',
-};
-
-function tagBill(policyArea) {
- const cat = POLICY_AREA_TO_CATEGORY[policyArea ?? ''];
- return cat ? [cat] : [];
-}
-
// ---- AI decode (new bills only) ----
async function fetchBillText(type, number) {
const data = await cg(`/bill/${CONGRESS}/${type}/${number}/text`);
@@ -258,6 +214,110 @@ const since = state.lastSync;
const runStart = new Date().toISOString();
console.log(`sync since ${since}`);
+// Shared new-bill decode-budget counter - both passes below decrement into
+// this ONE pool (RECENT_DECODE_RESERVE is a ceiling on the recent-first
+// pass's share of it, not a separate allowance; see the header comment).
+let added = 0;
+let refreshed = 0; // combined total across both passes (log-only, not gated)
+
+/**
+ * Fetch one bill's current detail and either refresh it (already in the
+ * corpus - free) or decode it as new (only if `allowDecode`). The one place
+ * both passes below do "turn a Congress.gov update item into a corpus
+ * mutation", so the decode-before-publish invariant and the refresh fields
+ * can't drift between the recent-first pass and the ascending backlog pass.
+ * Returns one of:
+ * 'refreshed' - an existing bill's fields were updated in place
+ * 'added' - a brand-new bill was decoded and pushed into the corpus
+ * 'budget' - a brand-new bill was found but `allowDecode` was false
+ * 'failed' - the fetch or decode threw; `isNew` tells the caller
+ * whether this was a new-bill decode failure (must retry)
+ * or an existing bill's transient refresh failure
+ * (idempotent, self-heals on its next update).
+ */
+async function syncOneBill(u, allowDecode) {
+ const type = u.type.toLowerCase();
+ const slug = updateSlug(u);
+ try {
+ const { bill: d } = await cg(`/bill/${CONGRESS}/${type}/${u.number}`);
+ const existing = bySlug.get(slug);
+ if (existing) {
+ refreshBillFields(existing, d);
+ return { outcome: 'refreshed', slug };
+ }
+ if (!allowDecode) return { outcome: 'budget', slug };
+ const status = mapStatus(d.latestAction?.text);
+ const lastActionDate = d.latestAction?.actionDate ?? null;
+ const bill = {
+ full_identifier: slug,
+ congress_number: CONGRESS,
+ bill_type: type,
+ bill_number: Number(u.number),
+ title: d.title,
+ short_title: null,
+ ai_summary: null, ai_headline: null,
+ sponsor_bioguide_id: d.sponsors?.[0]?.bioguideId ?? null,
+ introduced_date: d.introducedDate ?? null,
+ last_action_date: lastActionDate,
+ last_action_text: d.latestAction?.text ?? null,
+ status,
+ issue_tags: tagBill(d.policyArea?.name),
+ policy_area: d.policyArea?.name ?? null,
+ urgency_score: urgencyScore(status, lastActionDate),
+ congress_gov_url: `https://www.congress.gov/bill/${CONGRESS}th-congress/${type === 'hr' ? 'house-bill' : type === 's' ? 'senate-bill' : type === 'hjres' ? 'house-joint-resolution' : 'senate-joint-resolution'}/${u.number}`,
+ };
+ const text = await fetchBillText(type, u.number);
+ const dec = await decode(bill, text);
+ bill.ai_summary = dec.ai_summary;
+ bill.ai_headline = dec.ai_headline;
+ bill.ai_sections = dec.ai_sections;
+ // Search handles for the coverage sync (press names + subject query).
+ // Non-fatal: the backfill script sweeps up any misses.
+ try {
+ const si = await generateSearchInputs(anthropic, bill);
+ bill.press_names = si.press_names;
+ bill.news_query = si.news_query;
+ } catch (e) {
+ console.error(` search-inputs failed for ${slug}: ${e.message}`);
+ }
+ es[slug] = { headline: dec.es_headline, summary: dec.es_summary, sections: dec.es_sections };
+ bills.push(bill);
+ bySlug.set(slug, bill);
+ return { outcome: 'added', slug };
+ } catch (e) {
+ console.error(`FAIL ${slug}: ${e.message}`);
+ return { outcome: 'failed', slug, isNew: !bySlug.has(slug) };
+ }
+}
+
+// ---- Pass 1: recent-first (audit §5 item 2) ----------------------------
+// Guarantees this run always sees the most recently-touched bills in
+// Congress, no matter how deep the ascending backlog is. `handledSlugs`
+// tracks everything this pass successfully resolved so pass 2 can dedupe
+// without re-fetching or re-decoding - see updateSlug/refreshBillFields.
+const handledSlugs = new Set();
+const recentDecodeCap = Math.min(RECENT_DECODE_RESERVE, MAX_NEW_DECODES);
+console.log(`recent-first pass: fetching up to ${RECENT_FETCH_LIMIT} most-recently-updated bills (decode reserve ${recentDecodeCap})`);
+const recentBills = await fetchRecentlyUpdated(RECENT_FETCH_LIMIT);
+let recentRefreshed = 0, recentAdded = 0, recentDeferred = 0, recentFailed = 0;
+for (const u of recentBills) {
+ const result = await syncOneBill(u, added < recentDecodeCap);
+ if (result.outcome === 'refreshed') {
+ refreshed++; recentRefreshed++; handledSlugs.add(result.slug);
+ } else if (result.outcome === 'added') {
+ added++; recentAdded++; handledSlugs.add(result.slug);
+ } else if (result.outcome === 'budget') {
+ recentDeferred++; // new bill, reserve exhausted - left for pass 2 (same run) or next run
+ } else {
+ recentFailed++; // logged only; deliberately NOT folded into the abort check below
+ }
+}
+console.log(`recent-first pass: ${recentRefreshed} refreshed, ${recentAdded} added+decoded, ${recentDeferred} deferred (reserve exhausted), ${recentFailed} failed`);
+
+// ---- Pass 2: ascending backlog scan from the cursor ---------------------
+// Unchanged shape from before the two-pass fetch - see the header comment.
+// The freeze-on-incomplete-work cursor logic below is tied ONLY to this
+// pass; pass 1 above never touches `cursor`/`frozen`.
const updated = [];
let offset = 0;
for (;;) {
@@ -271,7 +331,7 @@ for (;;) {
}
console.log(`${updated.length} updated bills (capped at ${MAX_UPDATES})`);
-let refreshed = 0, added = 0, queued = 0, failed = 0;
+let queued = 0, failed = 0;
// High-water mark: advance the cursor over every bill we fully handle, and
// freeze it the instant we hit one that still needs work (decode budget
// exhausted, or a new bill whose decode failed). A transient *refresh* failure
@@ -281,70 +341,27 @@ let refreshed = 0, added = 0, queued = 0, failed = 0;
let cursor = since;
let frozen = false;
for (const u of updated.slice(0, MAX_UPDATES)) {
- const type = u.type.toLowerCase();
- const slug = `${type}-${u.number}-${CONGRESS}`;
+ const slug = updateSlug(u);
let needsWork = false;
- try {
- const { bill: d } = await cg(`/bill/${CONGRESS}/${type}/${u.number}`);
- const status = mapStatus(d.latestAction?.text);
- const lastActionDate = d.latestAction?.actionDate ?? null;
- const existing = bySlug.get(slug);
- if (existing) {
- existing.status = status;
- existing.last_action_date = lastActionDate;
- existing.last_action_text = d.latestAction?.text ?? existing.last_action_text;
- existing.urgency_score = urgencyScore(status, lastActionDate);
- const tags = tagBill(d.policyArea?.name);
- if (tags.length) existing.issue_tags = tags;
- existing.policy_area = d.policyArea?.name ?? existing.policy_area;
+ if (handledSlugs.has(slug)) {
+ // Already fully resolved by the recent-first pass this run - dedupe,
+ // don't re-fetch/re-decode. Resolved is resolved, so the cursor may
+ // still advance over it exactly as if pass 2 had handled it itself.
+ } else {
+ const result = await syncOneBill(u, added < MAX_NEW_DECODES);
+ if (result.outcome === 'refreshed') {
refreshed++;
- } else if (added < MAX_NEW_DECODES) {
- const bill = {
- full_identifier: slug,
- congress_number: CONGRESS,
- bill_type: type,
- bill_number: Number(u.number),
- title: d.title,
- short_title: null,
- ai_summary: null, ai_headline: null,
- sponsor_bioguide_id: d.sponsors?.[0]?.bioguideId ?? null,
- introduced_date: d.introducedDate ?? null,
- last_action_date: lastActionDate,
- last_action_text: d.latestAction?.text ?? null,
- status,
- issue_tags: tagBill(d.policyArea?.name),
- policy_area: d.policyArea?.name ?? null,
- urgency_score: urgencyScore(status, lastActionDate),
- congress_gov_url: `https://www.congress.gov/bill/${CONGRESS}th-congress/${type === 'hr' ? 'house-bill' : type === 's' ? 'senate-bill' : type === 'hjres' ? 'house-joint-resolution' : 'senate-joint-resolution'}/${u.number}`,
- };
- const text = await fetchBillText(type, u.number);
- const dec = await decode(bill, text);
- bill.ai_summary = dec.ai_summary;
- bill.ai_headline = dec.ai_headline;
- bill.ai_sections = dec.ai_sections;
- // Search handles for the coverage sync (press names + subject query).
- // Non-fatal: the backfill script sweeps up any misses.
- try {
- const si = await generateSearchInputs(anthropic, bill);
- bill.press_names = si.press_names;
- bill.news_query = si.news_query;
- } catch (e) {
- console.error(` search-inputs failed for ${slug}: ${e.message}`);
- }
- es[slug] = { headline: dec.es_headline, summary: dec.es_summary, sections: dec.es_sections };
- bills.push(bill);
- bySlug.set(slug, bill);
+ } else if (result.outcome === 'added') {
added++;
- } else {
+ } else if (result.outcome === 'budget') {
queued++; // decode budget exhausted; revisit next run
needsWork = true;
+ } else {
+ failed++;
+ // A new bill that failed to decode must be retried; a failed refresh of
+ // a known bill is idempotent and re-touches on its next update.
+ if (result.isNew) needsWork = true;
}
- } catch (e) {
- failed++;
- console.error(`FAIL ${slug}: ${e.message}`);
- // A new bill that failed to decode must be retried; a failed refresh of a
- // known bill is idempotent and re-touches on its next update.
- if (!bySlug.has(slug)) needsWork = true;
}
if (needsWork) frozen = true;
else if (!frozen && u.updateDate) cursor = toISODateTime(u.updateDate);
@@ -360,4 +377,8 @@ writeFileSync('data/bills.json', JSON.stringify(bills));
writeFileSync('data/bills-es.json', JSON.stringify(es));
writeFileSync('data/sync-state.json', JSON.stringify(state, null, 2));
console.log(`DONE: ${refreshed} refreshed, ${added} added+decoded, ${queued} queued for next run, ${failed} failed; corpus ${bills.length}`);
-if (failed > updated.length / 2) process.exit(1); // mostly-failed run: don't let CI commit garbage
+// Mostly-failed run: don't let CI commit garbage. Scoped to the ascending
+// pass's own failed/updated.length exactly as before the two-pass fetch -
+// the recent-first pass's (much smaller, logged-separately) failures don't
+// feed this check.
+if (failed > updated.length / 2) process.exit(1);
diff --git a/scripts/sync-coverage.mjs b/scripts/sync-coverage.mjs
index 02d5abc..233cfc2 100644
--- a/scripts/sync-coverage.mjs
+++ b/scripts/sync-coverage.mjs
@@ -24,10 +24,19 @@ if (!NEWS_API_KEY) {
process.exit(0);
}
-// Cover every eligible bill by default (the news API's daily quota is the real
-// ceiling; the run stops early and commits what it has if quota is hit). 25
-// candidates/bill is TheNewsAPI's Basic-tier per-request max.
-const TOP_N = Number(process.env.COVERAGE_TOP_N ?? Infinity);
+// Capped at the 150 most urgent eligible bills by default (raised from the
+// prior unbounded Infinity, 2026-07-16, audit §5 item 1): at Infinity this
+// queried every eligible bill in urgency order every night, so a bill sitting
+// near the bottom of the ranking was effectively guaranteed to lose the race
+// against TheNewsAPI's daily quota before it was ever reached - the same
+// bills, night after night, never got a fresh look. 150 keeps the nightly
+// query bounded while comfortably covering everything that could plausibly
+// clear the "Act now"/"Moving" floors (taxonomy.ts's BAND_SIZES is 18 total),
+// still overridable via env for a one-off wider sweep. The news API's daily
+// quota remains the real ceiling either way - the run stops early and commits
+// what it has if quota is hit. 25 candidates/bill is TheNewsAPI's Basic-tier
+// per-request max.
+const TOP_N = Number(process.env.COVERAGE_TOP_N ?? 150);
const PER_BILL = Number(process.env.COVERAGE_PER_BILL ?? 5);
const MAX_CANDIDATES = Number(process.env.COVERAGE_MAX_CANDIDATES ?? 25);
const NEWS_API = 'https://api.thenewsapi.com/v1/news/all';
diff --git a/scripts/verify-sync.mjs b/scripts/verify-sync.mjs
index c787a60..27a2f2f 100644
--- a/scripts/verify-sync.mjs
+++ b/scripts/verify-sync.mjs
@@ -11,21 +11,38 @@
* (RUN_STARTED_AT, captured by the workflow before the sync step)
* - lastSync is not a full ISO-8601 datetime — the bare-date cursor that
* 400-looped every night from 2026-06-25 to 07-01 (PR #16)
+ * - the sync cursor (lastSync) is more than CURSOR_MAX_AGE_DAYS old — see
+ * the "Cursor-age threshold" note below (2026-07-16, audit §5 item 4;
+ * promoted from a non-blocking ::warning)
* - the bill count dropped more than 2% vs the committed corpus (the sync
* only ever appends, so any real drop means corruption)
* - EN/ES parity broke: a decoded bill without a bills-es.json entry, or
* an ES entry pointing at a bill that doesn't exist
*
- * WARNS (::warning, never fails) on corpus staleness: the cursor is weeks
- * behind BY DESIGN while the 361-bill decode backlog drains (the high-water
+ * Cursor-age threshold (2026-07-16, audit §5 item 4). This check used to be
+ * a non-blocking ::warning, on the theory that the cursor would sit weeks
+ * behind BY DESIGN while a 361-bill decode backlog drained (the high-water
* mark freezes at the oldest bill still awaiting decode — see
- * docs/solutions/pinned-sync-cursor.md). A wall-clock staleness failure
- * would fire every night until the backlog clears; revisit the threshold
- * once these warnings stop.
+ * docs/solutions/pinned-sync-cursor.md). Live logs proved that premise
+ * false: the warning fired every clean night for weeks (06-16 through
+ * 07-14) and was never acted on — exactly the silent-failure shape this
+ * script exists to prevent, and the root cause behind "worth a call"
+ * reading stale/empty in production. Promoted to a hard failure, at a
+ * DELIBERATELY GENEROUS CURSOR_MAX_AGE_DAYS=10 (not the old 7-day warning
+ * threshold): the raised MAX_NEW_DECODES + the recent-first two-pass fetch
+ * (scripts/sync-bills.mjs) still need real nights to drain the pre-existing
+ * backlog once this change merges, and a threshold that insta-fails the
+ * very next run would block that catch-up window instead of giving it room
+ * to work. 10 days sits comfortably below lib/freshness-state.ts's
+ * FRESHNESS_DEAD_WINDOW_DAYS=21 (the site's own "this has gone genuinely
+ * dead" ceiling for the SAME cursor value), so CI catches a regression well
+ * before a visitor could ever see a dishonest "quiet week" from it.
*/
import { execSync } from 'node:child_process';
import { readFileSync } from 'node:fs';
+const CURSOR_MAX_AGE_DAYS = 10;
+
let failed = false;
const fail = (msg) => {
console.error(`::error::${msg}`);
@@ -140,9 +157,11 @@ if (state) {
);
} else {
const cursorAgeDays = (Date.now() - Date.parse(state.lastSync)) / 86_400_000;
- if (cursorAgeDays > 7) {
- warn(
- `corpus cursor is ${Math.round(cursorAgeDays)} days old (lastSync ${state.lastSync}). Expected while the decode backlog drains; if this warning persists after the backlog clears, promote it to a failure.`
+ // Hard failure, not a ::warning — see this file's header comment
+ // ("Cursor-age threshold") for why 10 days and why this was promoted.
+ if (cursorAgeDays > CURSOR_MAX_AGE_DAYS) {
+ fail(
+ `corpus cursor is ${Math.round(cursorAgeDays)} days old (lastSync ${state.lastSync}), past the ${CURSOR_MAX_AGE_DAYS}-day ceiling — the ascending backlog scan has stopped making real progress`
);
}
}
diff --git a/tests/freshness.spec.ts b/tests/freshness.spec.ts
index f421ef6..424a4e7 100644
--- a/tests/freshness.spec.ts
+++ b/tests/freshness.spec.ts
@@ -3,6 +3,7 @@ import syncState from '../data/sync-state.json';
import bills from '../data/bills.json';
import { TERMINAL_STATUSES, effectiveUrgency } from '../lib/urgency.mjs';
import { bandFloors } from '../lib/taxonomy';
+import { FRESHNESS_DEAD_WINDOW_DAYS, freshnessAgeDays } from '../lib/freshness-state';
/*
* KTD-1 / KTD-2 / AE3. The stamp is baked at build time from getFreshness()
@@ -41,6 +42,25 @@ const FRESH_CLOCK = LAST_RUN + 60 * 60 * 1000; // 1h after the last check
const STALE_CLOCK = LAST_RUN + 10 * 86_400_000; // past the 5d claim window
const DEAD_CLOCK = LAST_RUN + 30 * 86_400_000; // past the 21d dead window
+// 2026-07-16 (audit §5 item 4): emptyStateVerdict no longer looks only at
+// lastRun/checkedAt — the sync cursor (lastSync/completeThrough) and the
+// corpus's own newest last_action_date now independently gate the verdict
+// too (lib/freshness-state.ts). Mirror that here, corpus-derived exactly
+// like anyNow/anyTop above, rather than hardcoding today's specific data —
+// so these tests keep tracking the site's real behavior as the nightly sync
+// rewrites data/ instead of silently drifting from it.
+const newestActionDate = (bills as CorpusBill[]).reduce(
+ (max, b) => (b.last_action_date && b.last_action_date > max ? b.last_action_date : max),
+ ''
+);
+/** Whether the empty-state verdict reads data_stale AT FRESH_CLOCK for a
+ * reason that has nothing to do with lastRun (which is fresh by
+ * construction at FRESH_CLOCK) — i.e. the cursor or the corpus's newest
+ * known activity has gone dark past the dead window. */
+const contentStaleAtFreshClock =
+ freshnessAgeDays(syncState.lastSync, FRESH_CLOCK) > FRESHNESS_DEAD_WINDOW_DAYS ||
+ freshnessAgeDays(newestActionDate, FRESH_CLOCK) > FRESHNESS_DEAD_WINDOW_DAYS;
+
/** Collect hydration-related console errors — the AE3 client verdict must
* never be bought at the price of a server/client HTML mismatch. */
function trackHydrationErrors(page: Page): string[] {
@@ -76,19 +96,29 @@ test.describe('freshness stamp reads from sync-state via the shared accessor', (
});
test.describe('AE3: quiet-week vs data-stale tri-state (homepage)', () => {
- test('fresh clock: quiet week reads as quiet — and only on a truly quiet corpus', async ({ page }) => {
+ test('fresh clock: quiet week reads as quiet — and only on a truly quiet, genuinely current corpus', async ({ page }) => {
const hydrationErrors = trackHydrationErrors(page);
await page.clock.setFixedTime(FRESH_CLOCK);
await page.goto('/');
const quietCard = page.getByRole('status').filter({ hasText: /Quiet week/ });
+ const staleCard = page.getByRole('status').filter({ hasText: /Data check needed/ });
if (anyTop) {
// Hot corpus: cards render, no quiet-week claim anywhere.
await expect(
page.locator('section[aria-labelledby="top-actions"] a[href*="/bills/"]').first()
).toBeVisible();
await expect(quietCard).toHaveCount(0);
+ } else if (contentStaleAtFreshClock) {
+ // The band is empty, but the sync cursor or the corpus's own newest
+ // activity is dead-window-stale — never claim "quiet" over that, even
+ // though lastRun (checkedAt) itself is fresh at this clock (2026-07-16
+ // fix, audit §5 item 4: emptyStateVerdict no longer looks at lastRun
+ // alone).
+ await expect(staleCard).toBeVisible();
+ await expect(quietCard).toHaveCount(0);
} else if (!anyNow) {
- // Genuinely quiet: the honest empty state, never a padded card.
+ // Genuinely quiet AND genuinely current: the honest empty state, never
+ // a padded card.
await expect(quietCard).toBeVisible();
} else {
// Floor cleared only by undecoded bills: no cards, but also no false
@@ -102,7 +132,8 @@ test.describe('AE3: quiet-week vs data-stale tri-state (homepage)', () => {
test.skip(anyNow, 'corpus not quiet this week — ES quiet-week copy not renderable');
await page.clock.setFixedTime(FRESH_CLOCK);
await page.goto('/es');
- await expect(page.getByRole('status').filter({ hasText: 'Semana tranquila' })).toBeVisible();
+ const text = contentStaleAtFreshClock ? 'Verificación pendiente' : 'Semana tranquila';
+ await expect(page.getByRole('status').filter({ hasText: text })).toBeVisible();
});
test('stale clock: the empty slot says "data check needed", never "quiet"', async ({ page }) => {
@@ -130,10 +161,18 @@ test.describe('AE3: /bills "Act now" band mirrors the same tri-state', () => {
await page.clock.setFixedTime(FRESH_CLOCK);
await page.goto('/bills');
const quietCard = page.getByRole('status').filter({ hasText: /Quiet week/ });
+ const staleCard = page.getByRole('status').filter({ hasText: /Data check needed/ });
if (!anyNow) {
- // The unfiltered now band renders the quiet-week card under its header.
+ // The unfiltered now band renders the empty-state card under its
+ // header — quiet_week only when the cursor/corpus are also genuinely
+ // current at this clock (audit §5 item 4), data_stale otherwise.
await expect(page.locator('section[aria-labelledby="band-now"]').getByRole('status')).toBeVisible();
- await expect(quietCard).toBeVisible();
+ if (contentStaleAtFreshClock) {
+ await expect(staleCard).toBeVisible();
+ await expect(quietCard).toHaveCount(0);
+ } else {
+ await expect(quietCard).toBeVisible();
+ }
} else {
await expect(page.locator('section[aria-labelledby="band-now"] a[href*="/bills/"]').first()).toBeVisible();
await expect(quietCard).toHaveCount(0);
diff --git a/tests/freshness.unit.spec.ts b/tests/freshness.unit.spec.ts
index caecac1..d8562f2 100644
--- a/tests/freshness.unit.spec.ts
+++ b/tests/freshness.unit.spec.ts
@@ -5,6 +5,7 @@ import {
emptyStateVerdict,
freshnessAgeDays,
freshnessState,
+ type FreshnessSignals,
} from '../lib/freshness-state';
// One frozen clock for both the input timestamp and the function's `now`,
@@ -13,6 +14,16 @@ import {
const NOW = Date.now();
const daysAgo = (n: number) => new Date(NOW - n * 86_400_000).toISOString();
+// All three signals fresh by default; each test overrides only the one
+// signal it's exercising, so a failure clearly pins down which signal caused
+// the verdict to flip.
+const signals = (overrides: Partial