From df0fcfea2fcb75ec775b9805331867a4175ede99 Mon Sep 17 00:00:00 2001 From: Colby Maxwell Date: Thu, 16 Jul 2026 13:38:53 -0400 Subject: [PATCH 1/3] =?UTF-8?q?feat(sync):=20priority=20decode=20gate=20?= =?UTF-8?q?=E2=80=94=20only=20real=20legislative=20motion=20spends=20a=20d?= =?UTF-8?q?ecode?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Owner directive: reduce spend right now by focusing the AI decode pass on bills with real legislative motion, not the "majority… junk with high odds of never going anywhere." A brand-new bill now only spends a decode if mapStatus's output clears scripts/decode-gate.mjs's passesGate() (markup or later); mere "referred to committee" — sampled at 1,573/1,706 (92.2%) of today's 'committee'-status bills — does not. Extracted the shared decode-before-publish path (fetchBillText/decode/ syncOneBill) out of sync-bills.mjs into scripts/bill-decode.mjs, taking bills/es/bySlug/anthropic as explicit parameters instead of module-scope closures, so the newsdesk (next commit) can decode a press-triggered bill via the exact same path instead of a second copy of the prompts. Gate-skipped bills are NOT stored and count as fully handled: the ascending backlog pass's cursor advances past them exactly as if decoded, which drains the multi-week decode backlog nearly for free. If a gated bill later gets real motion, Congress.gov's own updateDate resurfaces it on a later run and the gate re-evaluates. FORCE_DECODE_SLUGS (env + workflow_dispatch input) bypasses the gate for specific slugs. MAX_NEW_DECODES reverts to a pure safety ceiling (120 -> 60) now that the gate does the real limiting, and per-run logging now reports new-bills-seen/gated/decoded so the digest-era numbers stay honest. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/sync-bills.yml | 10 +- scripts/bill-decode.mjs | 249 +++++++++++++++++++++++ scripts/decode-gate.mjs | 81 ++++++++ scripts/sync-bills.mjs | 329 ++++++++++--------------------- tests/decode-gate.unit.spec.ts | 57 ++++++ 5 files changed, 493 insertions(+), 233 deletions(-) create mode 100644 scripts/bill-decode.mjs create mode 100644 scripts/decode-gate.mjs create mode 100644 tests/decode-gate.unit.spec.ts diff --git a/.github/workflows/sync-bills.yml b/.github/workflows/sync-bills.yml index 6d96b3d..29fedcd 100644 --- a/.github/workflows/sync-bills.yml +++ b/.github/workflows/sync-bills.yml @@ -6,8 +6,11 @@ on: workflow_dispatch: inputs: max_new_decodes: - description: 'Max new bills to AI-decode this run' - default: '120' + description: 'Max new bills to AI-decode this run (safety ceiling — the priority gate does the real limiting)' + default: '60' + force_decode_slugs: + description: 'Comma-separated slugs to bypass the priority gate for (e.g. hr-1234-119,s-45-119)' + default: '' permissions: contents: write @@ -35,7 +38,8 @@ jobs: env: CONGRESS_API_KEY: ${{ secrets.CONGRESS_API_KEY }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - MAX_NEW_DECODES: ${{ inputs.max_new_decodes || '120' }} + MAX_NEW_DECODES: ${{ inputs.max_new_decodes || '60' }} + FORCE_DECODE_SLUGS: ${{ inputs.force_decode_slugs || '' }} run: node scripts/sync-bills.mjs - name: Backfill coverage search inputs # Drains press_names/news_query for bills decoded before search-input diff --git a/scripts/bill-decode.mjs b/scripts/bill-decode.mjs new file mode 100644 index 0000000..cae64ce --- /dev/null +++ b/scripts/bill-decode.mjs @@ -0,0 +1,249 @@ +/** + * Shared decode-before-publish + priority-gate resolution for ONE bill, + * used by BOTH scripts/sync-bills.mjs (nightly recent-first + ascending- + * backlog passes) and scripts/newsdesk.mjs (hourly headline-triggered + * resync, Part 2 of the 2026-07-16 spend-reduction pair). One copy so the + * gate, the FORCE_DECODE_SLUGS bypass, and the actual decode-before-publish + * AI calls can't drift between callers — same "one copy" discipline as + * lib/urgency.mjs's STATUS_BASE and congress-fetch.mjs's refreshBillFields. + * + * Extracted 2026-07-16 from what was previously sync-bills.mjs's own + * module-scope decode() + syncOneBill(): moving these here (as functions + * that take bills/es/bySlug/anthropic explicitly rather than closing over + * module-scope state) is what lets scripts/newsdesk.mjs decode a + * press-triggered new bill via the EXACT SAME decode-before-publish path + * the nightly sync uses, instead of maintaining a second copy of the + * summary/headline/ES prompts that could drift. + */ +import { readFileSync } from 'node:fs'; +import { + CONGRESS, + cg, + mapStatus, + refreshBillFields, + tagBill, + updateSlug, + urgencyScore, +} from './congress-fetch.mjs'; +import { passesGate } from './decode-gate.mjs'; +import { generateSearchInputs } from './search-inputs.mjs'; + +// Sonnet 5's tokenizer runs ~30% more tokens than 4.6 for the same text, so +// max_tokens caps on its calls are sized up accordingly; thinking is disabled +// explicitly because Sonnet 5 defaults it ON when the field is omitted, which +// would add unbounded thinking spend to batch calls. +export const DECODE_MODEL = 'claude-sonnet-5'; + +async function fetchBillText(type, number) { + const data = await cg(`/bill/${CONGRESS}/${type}/${number}/text`); + const versions = data.textVersions ?? []; + for (const v of [...versions].reverse()) { + const fmt = (v.formats ?? []).find((f) => f.type === 'Formatted Text'); + if (fmt?.url) { + const res = await fetch(fmt.url); + if (!res.ok) continue; + const html = await res.text(); + return html.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim().slice(0, 60_000); + } + } + return null; +} + +const DECODE_TAGS = [ + 'HEADLINE_EN', 'HEADLINE_ES', + 'TLDR', 'WHAT', 'WHO', 'WHY', 'COST', 'COST_CHIPS', + 'ES_TLDR', 'ES_WHAT', 'ES_WHO', 'ES_WHY', 'ES_COST', 'ES_COST_CHIPS', 'ES_SUMMARY', +]; + +function parseTagged(text) { + const out = {}; + for (let i = 0; i < DECODE_TAGS.length; i++) { + const tag = DECODE_TAGS[i]; + const start = text.indexOf(`[${tag}]`); + if (start === -1) throw new Error(`missing [${tag}]`); + const next = DECODE_TAGS.slice(i + 1) + .map((t) => text.indexOf(`[${t}]`)) + .filter((x) => x > start); + const end = next.length ? Math.min(...next) : text.length; + out[tag] = text.slice(start + tag.length + 2, end).trim(); + } + return out; +} + +const normCost = (s) => (s === 'NONE' || !s ? null : s); + +function normChips(s) { + if (s === 'NONE' || !s) return null; + const chips = s.split('|').map((c) => c.trim()).filter(Boolean); + if (chips.length < 1 || chips.length > 3 || chips.some((c) => c.length > 48)) return null; + return chips; +} + +async function decode(anthropic, bill, text) { + const sum = await anthropic.messages.create({ + model: DECODE_MODEL, max_tokens: 900, thinking: { type: 'disabled' }, + messages: [{ role: 'user', content: `Explain this congressional bill in plain language for an everyday US resident (8th-grade reading level). 2-3 short paragraphs: what it actually does, and who it affects. Strictly nonpartisan, no advocacy, no preamble, no markdown. + +Bill: ${bill.bill_type.toUpperCase()} ${bill.bill_number} — ${bill.title} + +Full text (may be truncated): +${text ?? bill.title}` }], + }); + const ai_summary = sum.content[0].text.trim(); + + const rest = await anthropic.messages.create({ + model: DECODE_MODEL, max_tokens: 3250, thinking: { type: 'disabled' }, + messages: [{ role: 'user', content: `From this plain-language bill summary, produce headlines, scannable sections, and a Spanish translation. + +Bill: ${bill.bill_type.toUpperCase()} ${bill.bill_number} +Summary: +${ai_summary} + +STRICT RULES: +- Use ONLY facts present in the summary. Never invent numbers, costs, or claims. +- Headlines: 45-90 chars, sentence case, factual news-desk style, varied construction (NOT "Topic — Consequence", avoid colons), never start with "Congress". Prioritize the most decision-relevant specifics: what it does, who it affects, what it costs, or where it stands. +- TLDR: one sentence, max 160 chars, the single most decision-relevant fact. +- WHAT: 1-3 sentences. WHO: 1-2. WHY: 1-2 sentences of neutral consequence, never benefits-framing. +- COST: 1-2 sentences ONLY if the summary contains spending/funding/fines/who-pays content; otherwise output exactly NONE (and ES_COST, COST_CHIPS, ES_COST_CHIPS all NONE too). +- COST_CHIPS: when COST exists, compress it to 2-3 chips separated by " | ", each a standalone fact fragment max 45 chars, sentence case, no period. Same count and order in ES_COST_CHIPS. If a fact can't fit 45 chars, output NONE for both chip tags (prose is the fallback). +- Spanish: natural Latin American Spanish, 8th-grade level; citations/numbers exact; agency names in English with a short gloss when helpful. ES_SUMMARY is the full summary translation. +- Plain text, no markdown. + +Output exactly this tagged format, each tag on its own line followed by its content: +[HEADLINE_EN] +[HEADLINE_ES] +[TLDR] +[WHAT] +[WHO] +[WHY] +[COST] +[COST_CHIPS] +[ES_TLDR] +[ES_WHAT] +[ES_WHO] +[ES_WHY] +[ES_COST] +[ES_COST_CHIPS] +[ES_SUMMARY]` }], + }); + const p = parseTagged(rest.content[0].text.trim()); + if (!p.HEADLINE_EN || !p.TLDR || !p.WHAT || !p.WHO || !p.WHY || !p.ES_SUMMARY) { + throw new Error('bad decode shape'); + } + return { + ai_summary, + ai_headline: p.HEADLINE_EN.slice(0, 110), + ai_sections: { + tldr: p.TLDR, what: p.WHAT, who: p.WHO, why: p.WHY, + cost: normCost(p.COST), costChips: normChips(p.COST_CHIPS), + }, + es_headline: p.HEADLINE_ES.slice(0, 110), + es_summary: p.ES_SUMMARY, + es_sections: { + tldr: p.ES_TLDR, what: p.ES_WHAT, who: p.ES_WHO, why: p.ES_WHY, + cost: normCost(p.ES_COST), costChips: normChips(p.ES_COST_CHIPS), + }, + }; +} + +/** + * Fetch one bill's current detail and either refresh it (already in the + * corpus — free, unconditional) or, for a brand-new bill, run it through + * the priority gate and decode-before-publish. The ONE place both + * sync-bills.mjs's passes and newsdesk.mjs's trigger path turn a + * Congress.gov update item ({type, number}) into a corpus mutation, so the + * gate, the force-bypass, and the refresh fields can't drift between + * callers. + * + * `u` is `{type, number}` (Congress.gov's shape, or newsdesk.mjs's own + * slug-derived equivalent). `ctx`: + * - allowDecode: this call may spend a decode if it clears the gate + * (the caller's own budget bookkeeping — MAX_NEW_DECODES for + * sync-bills.mjs, NEWSDESK_DECODE_CAP for newsdesk.mjs). + * - forceSlugs: a Set of slugs that bypass the priority gate entirely + * (still subject to allowDecode). Populated from FORCE_DECODE_SLUGS + * for manual/workflow_dispatch runs, or built in-process by + * newsdesk.mjs from headline-triggered bills — see decode-gate.mjs. + * - bills, es, bySlug, anthropic: the caller's loaded corpus + client. + * + * Returns one of: + * 'refreshed' — an existing bill's fields were updated in place (free) + * 'added' — a brand-new bill was decoded and pushed into the corpus + * 'gated' — a brand-new bill was found but shows no real legislative + * motion (and isn't force-bypassed) — NOT stored anywhere. + * Fully handled: if it later moves, Congress.gov's own + * updateDate advances past the caller's cursor and the + * update feed resurfaces it on a future run, when the gate + * re-evaluates against its then-current status. + * 'budget' — a brand-new bill cleared the gate (or was forced) but + * `allowDecode` was false this call + * 'failed' — the fetch or decode threw; `isNew` tells the caller + * whether this was a new-bill decode failure (must retry) + * or an existing bill's transient refresh failure + * (idempotent, self-heals on its next update). + */ +export async function syncOneBill(u, ctx) { + const { allowDecode, forceSlugs = new Set(), bills, es, bySlug, anthropic } = ctx; + const type = u.type.toLowerCase(); + const slug = updateSlug(u); + try { + const { bill: d } = await cg(`/bill/${CONGRESS}/${type}/${u.number}`); + const existing = bySlug.get(slug); + if (existing) { + refreshBillFields(existing, d); + return { outcome: 'refreshed', slug }; + } + const status = mapStatus(d.latestAction?.text); + const forced = forceSlugs.has(slug); + if (!forced && !passesGate(status)) { + return { outcome: 'gated', slug, status }; + } + if (!allowDecode) return { outcome: 'budget', slug }; + const lastActionDate = d.latestAction?.actionDate ?? null; + const bill = { + full_identifier: slug, + congress_number: CONGRESS, + bill_type: type, + bill_number: Number(u.number), + title: d.title, + short_title: null, + ai_summary: null, ai_headline: null, + sponsor_bioguide_id: d.sponsors?.[0]?.bioguideId ?? null, + introduced_date: d.introducedDate ?? null, + last_action_date: lastActionDate, + last_action_text: d.latestAction?.text ?? null, + status, + issue_tags: tagBill(d.policyArea?.name), + policy_area: d.policyArea?.name ?? null, + urgency_score: urgencyScore(status, lastActionDate), + congress_gov_url: `https://www.congress.gov/bill/${CONGRESS}th-congress/${type === 'hr' ? 'house-bill' : type === 's' ? 'senate-bill' : type === 'hjres' ? 'house-joint-resolution' : 'senate-joint-resolution'}/${u.number}`, + }; + const text = await fetchBillText(type, u.number); + const dec = await decode(anthropic, bill, text); + bill.ai_summary = dec.ai_summary; + bill.ai_headline = dec.ai_headline; + bill.ai_sections = dec.ai_sections; + // Search handles for the coverage sync (press names + subject query). + // Non-fatal: the backfill script sweeps up any misses. + try { + const si = await generateSearchInputs(anthropic, bill); + bill.press_names = si.press_names; + bill.news_query = si.news_query; + } catch (e) { + console.error(` search-inputs failed for ${slug}: ${e.message}`); + } + es[slug] = { headline: dec.es_headline, summary: dec.es_summary, sections: dec.es_sections }; + bills.push(bill); + bySlug.set(slug, bill); + return { outcome: 'added', slug }; + } catch (e) { + console.error(`FAIL ${slug}: ${e.message}`); + return { outcome: 'failed', slug, isNew: !bySlug.has(slug) }; + } +} + +/** Read+parse a data/*.json file — tiny shared helper so both callers open + * the corpus the same way. */ +export function loadJSON(path) { + return JSON.parse(readFileSync(path, 'utf8')); +} diff --git a/scripts/decode-gate.mjs b/scripts/decode-gate.mjs new file mode 100644 index 0000000..8e7a74e --- /dev/null +++ b/scripts/decode-gate.mjs @@ -0,0 +1,81 @@ +/** + * The priority decode gate (2026-07-16, owner directive: reduce spend right + * now by focusing the AI decode pass on a priority set of legislation — + * "what's up for a vote, what's in the news… the majority of the 2,147 + * bills is junk with high odds of never going anywhere"). Split into its + * own tiny, I/O-free module so the gate decision itself is directly + * unit-testable without mocking Congress.gov or Anthropic — see + * tests/decode-gate.unit.spec.ts. + * + * ---- Status distribution across the full 2,147-bill corpus, sampled + * 2026-07-16 (data/bills.json, mapStatus's output field — see + * scripts/congress-fetch.mjs) ---- + * committee: 1,706 (79.5%) + * floor_vote: 152 ( 7.1%) + * passed_chamber: 147 ( 6.8%) + * markup: 115 ( 5.4%) + * signed: 27 ( 1.3%) + * (conference, vetoed: 0 today — mapStatus supports both, neither is + * currently mapped onto any bill in the corpus) + * + * ---- The CRITICAL NUANCE: does 'committee' mean mere day-1 referral, or + * real committee action? ---- + * Sampled the 1,706 'committee'-status bills' last_action_text directly: + * 1,573/1,706 (92.2%) literally start with "Referred to the … Committee + * on …" — the automatic first action every single bill gets on + * introduction, zero legislative motion. The remaining 133 (7.8%) are + * genuine sub-committee activity (e.g. "Committee on Veterans' Affairs. + * Hearings held.", "Subcommittee Hearings Held", "Committee Consideration + * and Mark-up Session Held") that `mapStatus` happens to miscategorize as + * 'committee' instead of 'markup' — e.g. "Committee Consideration and + * Mark-up Session Held" doesn't match `text.includes('markup')` because of + * the hyphen in "Mark-up". `mapStatus` itself (scripts/congress-fetch.mjs) + * is out of scope for this change — this gate treats its 'committee' + * output as untrustworthy en masse rather than adding a second classifier + * here. + * + * ---- Chosen gate line ---- + * A bill passes ONLY if mapStatus returned something OTHER than + * 'committee' — markup / floor_vote / passed_chamber / conference / + * signed / vetoed all count as "real legislative motion" per the owner's + * directive; 'committee' does not, because it is dominated (92%) by mere + * referral. This intentionally gates out the 133 miscategorized + * real-action bills along with the 1,573 referral-only ones, trading a + * small amount of recall for a bright, cheap-to-reason-about line that + * needs no new classification surface. If that recall loss turns out to + * matter in practice, fixing the "Mark-up" hyphen gap in `mapStatus` is + * the correct follow-up, not a second condition here. + * + * At today's distribution, roughly 20.5% of bills (441/2,147) would clear + * this line — matching the owner's own framing that "the majority… is + * junk with high odds of never going anywhere." + */ + +export const GATE_PASS_STATUSES = new Set([ + 'markup', + 'floor_vote', + 'passed_chamber', + 'conference', + 'signed', + 'vetoed', +]); + +/** True if `status` (mapStatus's output) shows real legislative motion and + * should be allowed through the decode gate. */ +export function passesGate(status) { + return GATE_PASS_STATUSES.has(status); +} + +/** Parse a comma-separated slug list (FORCE_DECODE_SLUGS env, or a + * programmatically-built set) into a lower-cased Set. Bypasses the gate + * for exactly these slugs — used by workflow_dispatch manual runs and by + * scripts/newsdesk.mjs, which builds its own force set in-process from + * headline-matched bills rather than round-tripping through the env var. */ +export function parseForceSlugs(raw) { + return new Set( + String(raw ?? '') + .split(',') + .map((s) => s.trim().toLowerCase()) + .filter(Boolean) + ); +} diff --git a/scripts/sync-bills.mjs b/scripts/sync-bills.mjs index 8dd8c02..e594cfb 100644 --- a/scripts/sync-bills.mjs +++ b/scripts/sync-bills.mjs @@ -8,10 +8,38 @@ * * Policy: * - Existing bills: status/action/urgency/tags refresh freely (no AI cost). - * - NEW bills are decode-before-publish: they enter the corpus only once - * their EN+ES summary and headline exist, so the feed never shows - * undecoded entries. At most MAX_NEW_DECODES per run (cost ceiling); - * the rest wait for the next night. + * - NEW bills are decode-before-publish AND priority-gated: a new bill only + * spends a decode if it clears the priority gate (real legislative + * motion — see scripts/decode-gate.mjs) or is explicitly force-listed. + * Bills that clear the gate enter the corpus only once their EN+ES + * summary and headline exist, so the feed never shows undecoded entries. + * At most MAX_NEW_DECODES per run (cost ceiling); the rest wait for the + * next night. + * + * PRIORITY DECODE GATE (2026-07-16, owner directive: reduce spend, focus on + * a priority set of legislation — "the majority of the 2,147 bills is junk + * with high odds of never going anywhere"). A brand-new bill is decoded + * ONLY if `decode-gate.mjs`'s `passesGate(status)` says so (markup or + * later — NOT mere "referred to committee", which the gate treats as no + * real motion; see that module's header comment for the full status- + * distribution numbers and reasoning behind the line). This is enforced in + * ONE place, `bill-decode.mjs`'s `syncOneBill`, shared by BOTH the + * recent-first pass and the ascending backlog pass below, so the gate can't + * drift between them. Gate-skipped bills are NOT stored anywhere and count + * as fully handled: the ascending pass's cursor advances past them exactly + * as if they'd been decoded — this is what drains the multi-week decode + * backlog nearly for free, since ~80% of the corpus never had a real + * chance of clearing MAX_NEW_DECODES anyway. If a gated bill later gets + * real legislative motion, Congress.gov bumps its updateDate past + * wherever the cursor then sits, so the update feed resurfaces it on a + * later run and the gate re-evaluates against its new status — nothing + * about being gated out once is permanent. + * + * FORCE_DECODE_SLUGS (comma-separated slugs, e.g. "hr-1234-119,s-45-119") + * bypasses the gate for exactly those slugs — for a manual/workflow_dispatch + * catch-up run, or set in-process by scripts/newsdesk.mjs when a headline + * trigger decides a brand-new bill is newsworthy enough to decode outside + * the gate's own status-based test (see decode-gate.mjs's parseForceSlugs). * * Two-pass fetch (2026-07-16, audit §5 item 2). Congress.gov is queried * TWICE per run, in this order: @@ -19,13 +47,14 @@ * ~100 most-recently-touched bills in the whole 119th Congress, no * cursor floor. Already-known bills refresh for free; brand-new bills * decode within a RESERVED sub-budget (RECENT_DECODE_RESERVE, carved - * OUT of MAX_NEW_DECODES, not additional). This exists because the - * ascending backlog scan below structurally reaches the newest bills - * LAST - on a night with a deep backlog (or a busy legislative day) a - * floor vote that just happened would otherwise lose the race against - * both MAX_UPDATES and MAX_NEW_DECODES every single night, which is - * exactly how HR 7378 (and the whole "worth a call" feed) went stale - * for weeks even on clean, successful runs (see the audit). + * OUT of MAX_NEW_DECODES, not additional) AND must clear the priority + * gate above. This exists because the ascending backlog scan below + * structurally reaches the newest bills LAST - on a night with a deep + * backlog (or a busy legislative day) a floor vote that just happened + * would otherwise lose the race against both MAX_UPDATES and + * MAX_NEW_DECODES every single night, which is exactly how HR 7378 + * (and the whole "worth a call" feed) went stale for weeks even on + * clean, successful runs (see the audit). * 2. Ascending backlog: `fromDateTime: lastSync, sort=updateDate+asc` - * unchanged from before, drains the historical backlog oldest-first * with whatever decode budget the recent-first pass didn't use. A bill @@ -43,30 +72,29 @@ * exactly the failure this preserves the fix for. */ import Anthropic from '@anthropic-ai/sdk'; -import { readFileSync, writeFileSync } from 'node:fs'; +import { writeFileSync } from 'node:fs'; +import { loadJSON, syncOneBill } from './bill-decode.mjs'; import { BILL_TYPES, CONGRESS, cg, fetchRecentlyUpdated, - mapStatus, - refreshBillFields, slugOf, - tagBill, updateSlug, - urgencyScore, } from './congress-fetch.mjs'; -import { generateSearchInputs } from './search-inputs.mjs'; +import { parseForceSlugs } from './decode-gate.mjs'; const MAX_UPDATES = Number(process.env.MAX_UPDATES ?? 500); -// Raised 40 -> 120 (2026-07-16, audit §5 item 1): live nightly logs showed -// 373-418 bills/night needing decode against a 40-bill budget, pinning the -// ascending-pass cursor (state.lastSync) weeks behind and starving newer -// bills of decode slots night after night. 120 doesn't fully clear that -// inflow alone (~$8-14/night at $0.07-0.15/bill) - see the two-pass fetch -// design note above for the fix that stops recency from losing the race -// structurally, independent of how large the budget is. -const MAX_NEW_DECODES = Number(process.env.MAX_NEW_DECODES ?? 120); +// Lowered 120 -> 60 (2026-07-16, priority-decode-gate spec): with the gate +// above now doing the REAL limiting (only ~20.5% of bills - markup or +// later - are even eligible to spend a decode), MAX_NEW_DECODES reverts to +// a pure safety ceiling rather than the primary cost control it was when +// every new bill was decode-eligible. 60 comfortably covers a busy night's +// worth of genuinely-moving bills (441 gate-eligible bills total in the +// corpus today) without needing the 120 headroom that existed only to +// out-run an unfiltered ~373-418/night inflow of mostly just-introduced, +// zero-motion bills. +const MAX_NEW_DECODES = Number(process.env.MAX_NEW_DECODES ?? 60); // The recent-first pass's fetch window (audit §5 item 2 / §4 Alt A) - same // rough size as the twice-daily hot-bills.mjs refresh pass. const RECENT_FETCH_LIMIT = Number(process.env.RECENT_FETCH_LIMIT ?? 100); @@ -75,19 +103,20 @@ const RECENT_FETCH_LIMIT = Number(process.env.RECENT_FETCH_LIMIT ?? 100); // the last ~100 updates leaves the full MAX_NEW_DECODES for the ascending // backlog pass; a night with several leaves proportionally less. const RECENT_DECODE_RESERVE = Number(process.env.RECENT_DECODE_RESERVE ?? 20); +// See the header comment above and decode-gate.mjs. Empty by default. +const forceSlugs = parseForceSlugs(process.env.FORCE_DECODE_SLUGS); const anthropic = new Anthropic({ maxRetries: 8 }); -// Sonnet 5's tokenizer runs ~30% more tokens than 4.6 for the same text, so -// max_tokens caps on its calls are sized up accordingly; thinking is disabled -// explicitly because Sonnet 5 defaults it ON when the field is omitted, which -// would add unbounded thinking spend to batch calls. -const MODEL = 'claude-sonnet-5'; -const bills = JSON.parse(readFileSync('data/bills.json', 'utf8')); -const es = JSON.parse(readFileSync('data/bills-es.json', 'utf8')); -const state = JSON.parse(readFileSync('data/sync-state.json', 'utf8')); +const bills = loadJSON('data/bills.json'); +const es = loadJSON('data/bills-es.json'); +const state = loadJSON('data/sync-state.json'); const bySlug = new Map(bills.map((b) => [slugOf(b), b])); +if (forceSlugs.size) { + console.log(`FORCE_DECODE_SLUGS active (gate bypassed for): ${[...forceSlugs].join(', ')}`); +} + // Congress.gov's bill-list `updateDate` field is date-only (e.g. "2026-06-04"), // not a full timestamp. Persisting it as-is breaks the next run's fromDateTime // query, which Congress.gov 400s on - the 2026-06-25/07-01 outage. Always @@ -96,223 +125,48 @@ function toISODateTime(d) { return /T/.test(d) ? d : `${d}T00:00:00Z`; } -// ---- AI decode (new bills only) ---- -async function fetchBillText(type, number) { - const data = await cg(`/bill/${CONGRESS}/${type}/${number}/text`); - const versions = data.textVersions ?? []; - for (const v of [...versions].reverse()) { - const fmt = (v.formats ?? []).find((f) => f.type === 'Formatted Text'); - if (fmt?.url) { - const res = await fetch(fmt.url); - if (!res.ok) continue; - const html = await res.text(); - return html.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim().slice(0, 60_000); - } - } - return null; -} - -const DECODE_TAGS = [ - 'HEADLINE_EN', 'HEADLINE_ES', - 'TLDR', 'WHAT', 'WHO', 'WHY', 'COST', 'COST_CHIPS', - 'ES_TLDR', 'ES_WHAT', 'ES_WHO', 'ES_WHY', 'ES_COST', 'ES_COST_CHIPS', 'ES_SUMMARY', -]; - -function parseTagged(text) { - const out = {}; - for (let i = 0; i < DECODE_TAGS.length; i++) { - const tag = DECODE_TAGS[i]; - const start = text.indexOf(`[${tag}]`); - if (start === -1) throw new Error(`missing [${tag}]`); - const next = DECODE_TAGS.slice(i + 1) - .map((t) => text.indexOf(`[${t}]`)) - .filter((x) => x > start); - const end = next.length ? Math.min(...next) : text.length; - out[tag] = text.slice(start + tag.length + 2, end).trim(); - } - return out; -} - -const normCost = (s) => (s === 'NONE' || !s ? null : s); - -function normChips(s) { - if (s === 'NONE' || !s) return null; - const chips = s.split('|').map((c) => c.trim()).filter(Boolean); - if (chips.length < 1 || chips.length > 3 || chips.some((c) => c.length > 48)) return null; - return chips; -} - -async function decode(bill, text) { - const sum = await anthropic.messages.create({ - model: MODEL, max_tokens: 900, thinking: { type: 'disabled' }, - messages: [{ role: 'user', content: `Explain this congressional bill in plain language for an everyday US resident (8th-grade reading level). 2-3 short paragraphs: what it actually does, and who it affects. Strictly nonpartisan, no advocacy, no preamble, no markdown. - -Bill: ${bill.bill_type.toUpperCase()} ${bill.bill_number} — ${bill.title} - -Full text (may be truncated): -${text ?? bill.title}` }], - }); - const ai_summary = sum.content[0].text.trim(); - - const rest = await anthropic.messages.create({ - model: MODEL, max_tokens: 3250, thinking: { type: 'disabled' }, - messages: [{ role: 'user', content: `From this plain-language bill summary, produce headlines, scannable sections, and a Spanish translation. - -Bill: ${bill.bill_type.toUpperCase()} ${bill.bill_number} -Summary: -${ai_summary} - -STRICT RULES: -- Use ONLY facts present in the summary. Never invent numbers, costs, or claims. -- Headlines: 45-90 chars, sentence case, factual news-desk style, varied construction (NOT "Topic — Consequence", avoid colons), never start with "Congress". Prioritize the most decision-relevant specifics: what it does, who it affects, what it costs, or where it stands. -- TLDR: one sentence, max 160 chars, the single most decision-relevant fact. -- WHAT: 1-3 sentences. WHO: 1-2. WHY: 1-2 sentences of neutral consequence, never benefits-framing. -- COST: 1-2 sentences ONLY if the summary contains spending/funding/fines/who-pays content; otherwise output exactly NONE (and ES_COST, COST_CHIPS, ES_COST_CHIPS all NONE too). -- COST_CHIPS: when COST exists, compress it to 2-3 chips separated by " | ", each a standalone fact fragment max 45 chars, sentence case, no period. Same count and order in ES_COST_CHIPS. If a fact can't fit 45 chars, output NONE for both chip tags (prose is the fallback). -- Spanish: natural Latin American Spanish, 8th-grade level; citations/numbers exact; agency names in English with a short gloss when helpful. ES_SUMMARY is the full summary translation. -- Plain text, no markdown. - -Output exactly this tagged format, each tag on its own line followed by its content: -[HEADLINE_EN] -[HEADLINE_ES] -[TLDR] -[WHAT] -[WHO] -[WHY] -[COST] -[COST_CHIPS] -[ES_TLDR] -[ES_WHAT] -[ES_WHO] -[ES_WHY] -[ES_COST] -[ES_COST_CHIPS] -[ES_SUMMARY]` }], - }); - const p = parseTagged(rest.content[0].text.trim()); - if (!p.HEADLINE_EN || !p.TLDR || !p.WHAT || !p.WHO || !p.WHY || !p.ES_SUMMARY) { - throw new Error('bad decode shape'); - } - return { - ai_summary, - ai_headline: p.HEADLINE_EN.slice(0, 110), - ai_sections: { - tldr: p.TLDR, what: p.WHAT, who: p.WHO, why: p.WHY, - cost: normCost(p.COST), costChips: normChips(p.COST_CHIPS), - }, - es_headline: p.HEADLINE_ES.slice(0, 110), - es_summary: p.ES_SUMMARY, - es_sections: { - tldr: p.ES_TLDR, what: p.ES_WHAT, who: p.ES_WHO, why: p.ES_WHY, - cost: normCost(p.ES_COST), costChips: normChips(p.ES_COST_CHIPS), - }, - }; -} - // ---- main ---- const since = state.lastSync; const runStart = new Date().toISOString(); console.log(`sync since ${since}`); -// Shared new-bill decode-budget counter - both passes below decrement into -// this ONE pool (RECENT_DECODE_RESERVE is a ceiling on the recent-first -// pass's share of it, not a separate allowance; see the header comment). +// Shared new-bill decode-budget counter and gate counter - both passes +// below decrement/increment into these ONE pools (RECENT_DECODE_RESERVE is +// a ceiling on the recent-first pass's share of `added`, not a separate +// allowance; see the header comment). let added = 0; let refreshed = 0; // combined total across both passes (log-only, not gated) +let gated = 0; // combined total across both passes - no real legislative motion +let newFailed = 0; // new-bill decode failures specifically (subset of `failed` below) -/** - * Fetch one bill's current detail and either refresh it (already in the - * corpus - free) or decode it as new (only if `allowDecode`). The one place - * both passes below do "turn a Congress.gov update item into a corpus - * mutation", so the decode-before-publish invariant and the refresh fields - * can't drift between the recent-first pass and the ascending backlog pass. - * Returns one of: - * 'refreshed' - an existing bill's fields were updated in place - * 'added' - a brand-new bill was decoded and pushed into the corpus - * 'budget' - a brand-new bill was found but `allowDecode` was false - * 'failed' - the fetch or decode threw; `isNew` tells the caller - * whether this was a new-bill decode failure (must retry) - * or an existing bill's transient refresh failure - * (idempotent, self-heals on its next update). - */ -async function syncOneBill(u, allowDecode) { - const type = u.type.toLowerCase(); - const slug = updateSlug(u); - try { - const { bill: d } = await cg(`/bill/${CONGRESS}/${type}/${u.number}`); - const existing = bySlug.get(slug); - if (existing) { - refreshBillFields(existing, d); - return { outcome: 'refreshed', slug }; - } - if (!allowDecode) return { outcome: 'budget', slug }; - const status = mapStatus(d.latestAction?.text); - const lastActionDate = d.latestAction?.actionDate ?? null; - const bill = { - full_identifier: slug, - congress_number: CONGRESS, - bill_type: type, - bill_number: Number(u.number), - title: d.title, - short_title: null, - ai_summary: null, ai_headline: null, - sponsor_bioguide_id: d.sponsors?.[0]?.bioguideId ?? null, - introduced_date: d.introducedDate ?? null, - last_action_date: lastActionDate, - last_action_text: d.latestAction?.text ?? null, - status, - issue_tags: tagBill(d.policyArea?.name), - policy_area: d.policyArea?.name ?? null, - urgency_score: urgencyScore(status, lastActionDate), - congress_gov_url: `https://www.congress.gov/bill/${CONGRESS}th-congress/${type === 'hr' ? 'house-bill' : type === 's' ? 'senate-bill' : type === 'hjres' ? 'house-joint-resolution' : 'senate-joint-resolution'}/${u.number}`, - }; - const text = await fetchBillText(type, u.number); - const dec = await decode(bill, text); - bill.ai_summary = dec.ai_summary; - bill.ai_headline = dec.ai_headline; - bill.ai_sections = dec.ai_sections; - // Search handles for the coverage sync (press names + subject query). - // Non-fatal: the backfill script sweeps up any misses. - try { - const si = await generateSearchInputs(anthropic, bill); - bill.press_names = si.press_names; - bill.news_query = si.news_query; - } catch (e) { - console.error(` search-inputs failed for ${slug}: ${e.message}`); - } - es[slug] = { headline: dec.es_headline, summary: dec.es_summary, sections: dec.es_sections }; - bills.push(bill); - bySlug.set(slug, bill); - return { outcome: 'added', slug }; - } catch (e) { - console.error(`FAIL ${slug}: ${e.message}`); - return { outcome: 'failed', slug, isNew: !bySlug.has(slug) }; - } -} +const ctxBase = { bills, es, bySlug, anthropic, forceSlugs }; // ---- Pass 1: recent-first (audit §5 item 2) ---------------------------- // Guarantees this run always sees the most recently-touched bills in // Congress, no matter how deep the ascending backlog is. `handledSlugs` -// tracks everything this pass successfully resolved so pass 2 can dedupe -// without re-fetching or re-decoding - see updateSlug/refreshBillFields. +// tracks everything this pass fully resolved (refreshed, added, OR gated - +// a gate verdict is a resolution too) so pass 2 can dedupe without +// re-fetching or re-deciding - see updateSlug/refreshBillFields. const handledSlugs = new Set(); const recentDecodeCap = Math.min(RECENT_DECODE_RESERVE, MAX_NEW_DECODES); console.log(`recent-first pass: fetching up to ${RECENT_FETCH_LIMIT} most-recently-updated bills (decode reserve ${recentDecodeCap})`); const recentBills = await fetchRecentlyUpdated(RECENT_FETCH_LIMIT); -let recentRefreshed = 0, recentAdded = 0, recentDeferred = 0, recentFailed = 0; +let recentRefreshed = 0, recentAdded = 0, recentGated = 0, recentDeferred = 0, recentFailed = 0; for (const u of recentBills) { - const result = await syncOneBill(u, added < recentDecodeCap); + const result = await syncOneBill(u, { ...ctxBase, allowDecode: added < recentDecodeCap }); if (result.outcome === 'refreshed') { refreshed++; recentRefreshed++; handledSlugs.add(result.slug); } else if (result.outcome === 'added') { added++; recentAdded++; handledSlugs.add(result.slug); + } else if (result.outcome === 'gated') { + gated++; recentGated++; handledSlugs.add(result.slug); } else if (result.outcome === 'budget') { - recentDeferred++; // new bill, reserve exhausted - left for pass 2 (same run) or next run + recentDeferred++; // new bill, gate cleared but reserve exhausted - left for pass 2 (same run) or next run } else { recentFailed++; // logged only; deliberately NOT folded into the abort check below } } -console.log(`recent-first pass: ${recentRefreshed} refreshed, ${recentAdded} added+decoded, ${recentDeferred} deferred (reserve exhausted), ${recentFailed} failed`); +console.log(`recent-first pass: ${recentRefreshed} refreshed, ${recentAdded} added+decoded, ${recentGated} gated (no real motion), ${recentDeferred} deferred (reserve exhausted), ${recentFailed} failed`); // ---- Pass 2: ascending backlog scan from the cursor --------------------- // Unchanged shape from before the two-pass fetch - see the header comment. @@ -336,8 +190,13 @@ let queued = 0, failed = 0; // freeze it the instant we hit one that still needs work (decode budget // exhausted, or a new bill whose decode failed). A transient *refresh* failure // on a bill already in the corpus is idempotent and self-heals on its next -// update, so it doesn't freeze us - the old all-or-nothing freeze is what -// pinned lastSync for weeks and turned every run into a full window re-scan. +// update, so it doesn't freeze us. A GATED bill is likewise fully handled +// (not "still needs work") - it's deliberately not stored, and re-enters +// naturally via Congress.gov's own updateDate if it later moves - so it +// advances the cursor too. This dual property (transient-refresh-failure +// tolerance + gate-skip-is-handled) is what drains the backlog fast instead +// of freezing on the ~80% of bills that were never going to clear the gate +// anyway. let cursor = since; let frozen = false; for (const u of updated.slice(0, MAX_UPDATES)) { @@ -345,14 +204,16 @@ for (const u of updated.slice(0, MAX_UPDATES)) { let needsWork = false; if (handledSlugs.has(slug)) { // Already fully resolved by the recent-first pass this run - dedupe, - // don't re-fetch/re-decode. Resolved is resolved, so the cursor may + // don't re-fetch/re-decide. Resolved is resolved, so the cursor may // still advance over it exactly as if pass 2 had handled it itself. } else { - const result = await syncOneBill(u, added < MAX_NEW_DECODES); + const result = await syncOneBill(u, { ...ctxBase, allowDecode: added < MAX_NEW_DECODES }); if (result.outcome === 'refreshed') { refreshed++; } else if (result.outcome === 'added') { added++; + } else if (result.outcome === 'gated') { + gated++; // real legislative motion absent - fully handled, NOT queued/frozen } else if (result.outcome === 'budget') { queued++; // decode budget exhausted; revisit next run needsWork = true; @@ -360,7 +221,7 @@ for (const u of updated.slice(0, MAX_UPDATES)) { failed++; // A new bill that failed to decode must be retried; a failed refresh of // a known bill is idempotent and re-touches on its next update. - if (result.isNew) needsWork = true; + if (result.isNew) { needsWork = true; newFailed++; } } } if (needsWork) frozen = true; @@ -376,7 +237,15 @@ state.lastRun = runStart; writeFileSync('data/bills.json', JSON.stringify(bills)); writeFileSync('data/bills-es.json', JSON.stringify(es)); writeFileSync('data/sync-state.json', JSON.stringify(state, null, 2)); -console.log(`DONE: ${refreshed} refreshed, ${added} added+decoded, ${queued} queued for next run, ${failed} failed; corpus ${bills.length}`); + +// New bills seen this run, deduped across both passes: every new-bill slug +// this run touched resolves to exactly one of added/gated/queued/newFailed +// by the time we get here (a pass-1 'budget' deferral that pass 2 later +// resolves is NOT double-counted - see recentDeferred's comment above). +const newSeen = added + gated + queued + newFailed; +console.log( + `DONE: ${refreshed} refreshed, ${added} added+decoded, ${gated} gated (no real legislative motion), ${queued} queued for next run, ${failed} failed (${newFailed} new); new bills seen this run: ${newSeen}; corpus ${bills.length}` +); // Mostly-failed run: don't let CI commit garbage. Scoped to the ascending // pass's own failed/updated.length exactly as before the two-pass fetch - // the recent-first pass's (much smaller, logged-separately) failures don't diff --git a/tests/decode-gate.unit.spec.ts b/tests/decode-gate.unit.spec.ts new file mode 100644 index 0000000..bbb4a18 --- /dev/null +++ b/tests/decode-gate.unit.spec.ts @@ -0,0 +1,57 @@ +import { expect, test } from '@playwright/test'; +// Pure, I/O-free module (no CONGRESS_API_KEY/ANTHROPIC_API_KEY needed) - see +// scripts/decode-gate.mjs's header comment for the full status-distribution +// numbers and the reasoning behind the chosen gate line. +import { GATE_PASS_STATUSES, parseForceSlugs, passesGate } from '../scripts/decode-gate.mjs'; + +/* + * These tests PIN the priority decode gate's status table. If a status here + * surprises you, the gate line changed - retune deliberately and update the + * distribution-numbers comment in decode-gate.mjs alongside the pin. + */ + +test.describe('passesGate (status table)', () => { + test('committee does NOT pass - mere referral dominates this bucket (92.2% sampled)', () => { + expect(passesGate('committee')).toBe(false); + }); + + test('markup and later all pass - real legislative motion', () => { + expect(passesGate('markup')).toBe(true); + expect(passesGate('floor_vote')).toBe(true); + expect(passesGate('passed_chamber')).toBe(true); + expect(passesGate('conference')).toBe(true); + expect(passesGate('signed')).toBe(true); + expect(passesGate('vetoed')).toBe(true); + }); + + test('unknown/introduced status does not pass', () => { + expect(passesGate('introduced')).toBe(false); + expect(passesGate('some_future_status')).toBe(false); + expect(passesGate('')).toBe(false); + expect(passesGate(undefined)).toBe(false); + }); + + test('GATE_PASS_STATUSES is exactly the 6 real-motion statuses', () => { + expect([...GATE_PASS_STATUSES].sort()).toEqual( + ['conference', 'floor_vote', 'markup', 'passed_chamber', 'signed', 'vetoed'] + ); + }); +}); + +test.describe('parseForceSlugs', () => { + test('parses a comma-separated list, trims and lower-cases', () => { + expect(parseForceSlugs('HR-1234-119, s-45-119 ,hjres-9-119')).toEqual( + new Set(['hr-1234-119', 's-45-119', 'hjres-9-119']) + ); + }); + + test('empty/undefined/whitespace-only input yields an empty set', () => { + expect(parseForceSlugs(undefined)).toEqual(new Set()); + expect(parseForceSlugs('')).toEqual(new Set()); + expect(parseForceSlugs(' ')).toEqual(new Set()); + }); + + test('drops empty entries from stray commas', () => { + expect(parseForceSlugs('hr-1-119,,s-2-119,')).toEqual(new Set(['hr-1-119', 's-2-119'])); + }); +}); From db8287b89181b24f438c5b580ec461d802e078f0 Mon Sep 17 00:00:00 2001 From: Colby Maxwell Date: Thu, 16 Jul 2026 13:39:11 -0400 Subject: [PATCH 2/3] feat(newsdesk): hourly free-RSS headline trigger, corroboration-gated MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New scripts/newsdesk.mjs + .github/workflows/newsdesk.yml (hourly, cron '7 * * * *'), the news-headline trigger half of the owner's spend-reduction ask, ALL-IN cost under $5/day. Free RSS only — NO paid news API (TheNewsAPI stays sync-coverage.mjs's own quota) — a politically-balanced 6-feed basket (The Hill/Roll Call/NPR/Fox/CBS + a Google News congressional-keyword query carrying per-article outlet attribution), each verified live 2026-07-16 to return real, parseable RSS/Atom. Matching, cheapest first (scripts/newsdesk-match.mjs, a pure/I-O-free module so it's unit-testable with zero mocking): t1 explicit bill-number citation regex (hr/s/hjres/sjres only — "H. Res. 12" and "US 567" correctly excluded) -> t2 free normalized-token overlap against corpus titles+press_names -> t3 ONE batched claude-haiku-4-5-20251001 call for headlines t2 leaves ambiguous, skipped entirely when that batch is empty. Nonpartisan guardrail (non-negotiable): a bill fires only on (a) an explicit citation from any outlet, or (b) a t2/t3 match corroborated by >=2 distinct outlets, accumulated across runs — single-outlet soft-match triggering would be a prioritization channel data/media-bias.json's display-only lean normalization doesn't cover. On fire, the bill refreshes (free) or, if not yet in the corpus, decodes via the exact same decode-before-publish path as sync-bills.mjs (force-bypassing the priority gate — the press trigger's own corroboration is the worthiness signal here), bounded by both NEWSDESK_DECODE_CAP=3/run and a NEWSDESK_DAILY_DECODE_CAP=10/day so the documented <$2/day ceiling is code-enforced, not just expected. A seen-headlines cache (actions/cache, GitHub's own 7-day-unused eviction) dedupes across hourly runs and degrades gracefully on a cache miss. Never writes data/coverage.json (stays sync-coverage.mjs's) or sync-state.json's nightly cursor. Commits data/ only when scripts/newsdesk.mjs actually changed something, so a no-op hour never triggers a deploy. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/newsdesk.yml | 119 +++++++++++ scripts/newsdesk-match.mjs | 287 +++++++++++++++++++++++++ scripts/newsdesk.mjs | 333 ++++++++++++++++++++++++++++++ tests/newsdesk-match.unit.spec.ts | 215 +++++++++++++++++++ 4 files changed, 954 insertions(+) create mode 100644 .github/workflows/newsdesk.yml create mode 100644 scripts/newsdesk-match.mjs create mode 100644 scripts/newsdesk.mjs create mode 100644 tests/newsdesk-match.unit.spec.ts diff --git a/.github/workflows/newsdesk.yml b/.github/workflows/newsdesk.yml new file mode 100644 index 0000000..968b208 --- /dev/null +++ b/.github/workflows/newsdesk.yml @@ -0,0 +1,119 @@ +name: Newsdesk headline trigger + +# Hourly headline-triggered bill resync (Part 2 of the 2026-07-16 +# spend-reduction pair; Part 1 is the priority decode gate in +# scripts/sync-bills.mjs). Polls a politically-balanced basket of free RSS +# feeds (NO paid news API - see scripts/newsdesk.mjs's header comment), +# matches headlines to bills cheapest-tier-first, and only acts on a bill +# that clears the >=2-distinct-outlet corroboration rule (or carries an +# explicit bill-number citation, which needs no corroboration). +# +# COST: expected ~$0.12/day Haiku (t3 disambiguation - most hourly runs' +# batch is empty, $0) + $0-0.45/day trigger decodes on a typical/busy day; +# NEWSDESK_DAILY_DECODE_CAP=10 (scripts/newsdesk.mjs) makes the documented +# <$2/day ceiling code-enforced, not just expected. Actions minutes: $0 +# (public repo, unlimited free minutes on standard runners). Full +# typical/busy/hard-ceiling breakdown in scripts/newsdesk.mjs's header +# comment and the introducing PR's report. + +on: + schedule: + - cron: '7 * * * *' # hourly, offset from the top of the hour + workflow_dispatch: + +permissions: + contents: write + +concurrency: + # Shared with sync-bills.yml/hot-bills.yml/refresh-legislators.yml so this + # can never race any of their commits to data/bills.json or data/bills-es.json. + group: data-sync + cancel-in-progress: false + +jobs: + newsdesk: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-node@v6 + # Needs npm ci (not stdlib-only like hot-bills.yml): scripts/newsdesk.mjs + # constructs its own Anthropic client for the t3 disambiguation call + # and the (shared, bill-decode.mjs) decode-before-publish path. + with: + node-version: 24 + cache: npm + - run: npm ci + - name: Restore seen-headlines cache + # Ever-growing-cache pattern (GitHub Actions caches can't be + # overwritten under the same key): restore-keys prefix-matches the + # most recent previous run's exact key; the save step below always + # writes a NEW run-scoped key. GitHub auto-evicts any cache entry + # unused for 7 days, which is exactly the "7-day retention" the spec + # asks for - no explicit TTL bookkeeping needed in the script itself. + # A cache miss (first run ever, or a fully-evicted history) degrades + # gracefully - see scripts/newsdesk.mjs's loadCache(). + uses: actions/cache/restore@v4 + with: + path: .newsdesk-cache + key: newsdesk-seen-${{ github.run_id }} + restore-keys: | + newsdesk-seen- + - name: Run newsdesk + env: + CONGRESS_API_KEY: ${{ secrets.CONGRESS_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + run: node scripts/newsdesk.mjs + - name: Save seen-headlines cache + # Always saves under this run's own unique key (see the restore + # step's comment) - if: always() so a mid-run failure still persists + # whatever cache state was written before the failure, rather than + # losing dedupe progress on every transient error. + if: always() + uses: actions/cache/save@v4 + with: + path: .newsdesk-cache + key: newsdesk-seen-${{ github.run_id }} + - name: Commit data + id: commit + # Same oravan-sync author + push-with-rebase-retry pattern as + # sync-bills.yml/hot-bills.yml - the author email must map to the + # Vercel-linked GitHub account or Vercel BLOCKs the auto-deploy + # (docs/solutions/vercel-bot-push-blocked-deploys.md). Commits + # ONLY when scripts/newsdesk.mjs actually changed data/ - an + # hourly no-op run (the common case) must never produce a deploy. + run: | + git config user.name "oravan-sync" + git config user.email "223600121+cm2489@users.noreply.github.com" + git add data/ + if git diff --cached --quiet; then + echo "no changes" + echo "changed=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + echo "changed=true" >> "$GITHUB_OUTPUT" + git commit -m "chore(data): newsdesk trigger $(date -u +%FT%HZ)" + # This job only ever touches data/bills.json/data/bills-es.json + # (no sync-state.json, no coverage.json - see scripts/newsdesk.mjs's + # Boundaries section) - disjoint from refresh-legislators.yml's + # files but overlapping sync-bills.yml's/hot-bills.yml's, all of + # which share this concurrency group. Rebase onto the latest main + # and retry, same pattern as those workflows. + for i in 1 2 3 4 5; do + if git push; then + echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + exit 0 + fi + echo "push rejected, rebasing onto latest main (attempt $i)" + git fetch origin main + git rebase origin/main + done + exit 1 + - name: Verify the deploy landed + # Same dead-man's-switch as sync-bills.yml/hot-bills.yml - poll + # production until it serves the SHA we just pushed. Skips with a + # notice until the PROD_URL repo variable is set. + if: steps.commit.outputs.changed == 'true' + env: + PROD_URL: ${{ vars.PROD_URL }} + EXPECT_SHA: ${{ steps.commit.outputs.sha }} + run: node scripts/verify-deploy.mjs diff --git a/scripts/newsdesk-match.mjs b/scripts/newsdesk-match.mjs new file mode 100644 index 0000000..52a71e5 --- /dev/null +++ b/scripts/newsdesk-match.mjs @@ -0,0 +1,287 @@ +/** + * Pure headline<->bill matching logic for scripts/newsdesk.mjs (Part 2 of + * the 2026-07-16 spend-reduction pair). Deliberately has ZERO imports of + * congress-fetch.mjs (which throws at import time without CONGRESS_API_KEY + * set) or '@anthropic-ai/sdk' — every function here is a plain string/data + * transform, so tests/newsdesk-match.unit.spec.ts can exercise the whole + * matching design (citation regex, local token overlap, the ≥2-outlet + * corroboration rule, feed parsing, the no-change-no-commit guard) with + * zero mocking and zero live network/API calls. + * + * ---- Three-tier match design, cheapest first ---- + * t1 (findCitations): regex over explicit bill-number citations + * ("H.R. 1234", "S. 567", "H.J.Res. 45"). Free, resolves directly to a + * slug for any of our four tracked types (hr, s, hjres, sjres; the + * 119th Congress). Fires on ANY single outlet — an explicit citation is + * unambiguous, so no corroboration is required. + * t2 (matchLocal): free normalized-token overlap against the corpus's own + * bill titles + press_names (data/bills.json fields — there is no + * separate data/search-inputs.json; press_names/news_query live + * directly on each bill object). A confident single match skips the LLM + * entirely; an ambiguous shortlist (multiple plausible candidates, or a + * weak-but-present signal on a legislative-looking headline) is handed + * to t3. + * t3 (resolved by scripts/newsdesk.mjs's one batched Haiku call): ONLY + * headlines t2 left ambiguous. This module supplies the batch + * membership test (looksLegislative) and validates the LLM's output + * against the offered candidates (a hallucinated slug is never trusted + * — see newsdesk.mjs's resolveWithHaiku). + * + * ---- The ≥2-outlet corroboration rule (decideFires) ---- + * A bill fires only if (a) an explicit citation matched it from ANY + * outlet, or (b) t2/t3 matched it from at least 2 DISTINCT outlets. (a) + * needs no corroboration because a citation is unambiguous. (b) does, + * because a free-text/LLM match to a bill's title is inherently softer, + * and — the nonpartisan guardrail this exists for — letting a single + * outlet's coverage alone decide which bills get fast-tracked ahead of + * others would make whichever outlet happens to publish first a de facto + * prioritization channel. data/media-bias.json's AllSides lean data + * already normalizes DISPLAY of an outlet's lean; it does nothing to stop + * a single-source story from silently jumping a bill to the front of the + * decode/refresh queue. Requiring 2 distinct outlets before a soft match + * can trigger anything makes that channel much harder to game with one + * placement, without blocking a bill that's genuinely breaking (which + * will show up in the citation tier, or in >1 outlet's feed within the + * same rolling window, almost immediately). + */ +import { createHash } from 'node:crypto'; + +// Duplicated from congress-fetch.mjs's CONGRESS constant (not imported) so +// this module stays import-clean for unit tests — see the header comment. +// The 119th Congress; bump alongside congress-fetch.mjs's own CONGRESS if +// the tracked Congress ever changes. +const CONGRESS = 119; +export const TRACKED_TYPES = new Set(['hr', 's', 'hjres', 'sjres']); + +// ---- t1: explicit bill-number citations ------------------------------ +// Each alternative requires the number to be IMMEDIATELY adjacent (through +// only an optional period and a single optional space) to the type token, +// which is what correctly rejects "H. Res. 12" (a simple House resolution +// — NOT one of our 4 tracked types; "Res" inserts non-dot/space characters +// between "H"/"R" and the digits, so no alternative can complete a match) +// and "US 567" (the leading \b can't fire inside "US" — no word boundary +// between "U" and "S"). HJRES/SJRES are tried before the shorter HR/S +// alternatives at each scan position so "H.J.Res. 45" resolves as hjres, +// not as a stray "H." partial. +const CITATION_RE = /\b(H\.?\s?J\.?\s?Res\.?|S\.?\s?J\.?\s?Res\.?|H\.?\s?R\.?|S\.?)\s?(\d{1,5})\b/gi; + +function normalizeType(raw) { + return raw.replace(/[^a-zA-Z]/g, '').toLowerCase(); +} + +const TYPE_ALIASES = { h: null, hr: 'hr', s: 's', hjres: 'hjres', sjres: 'sjres' }; + +/** Find every explicit, trackable bill-number citation in `text`. Returns + * `[{type, number, slug}]` — type is one of hr/s/hjres/sjres, slug is + * `${type}-${number}-119`. Citations to untracked types (e.g. "H. Res. + * 12", a simple resolution) are silently excluded, not returned as a + * partial/wrong match. */ +export function findCitations(text) { + const out = []; + const seen = new Set(); + for (const m of String(text ?? '').matchAll(CITATION_RE)) { + const type = TYPE_ALIASES[normalizeType(m[1])]; + if (!type) continue; + const number = String(Number(m[2])); // normalize away leading zeros, if any + const slug = `${type}-${number}-${CONGRESS}`; + if (seen.has(slug)) continue; + seen.add(slug); + out.push({ type, number, slug }); + } + return out; +} + +// ---- t2: free local token-overlap match ------------------------------- +const STOPWORDS = new Set([ + 'the', 'a', 'an', 'of', 'to', 'for', 'and', 'or', 'in', 'on', 'at', 'by', + 'with', 'from', 'into', 'act', 'acts', 'bill', 'bills', 'amendment', + 'amendments', 'congress', 'congressional', 'united', 'states', 'american', + 'establish', 'establishing', 'establishment', 'require', 'requiring', + 'provide', 'providing', 'relating', 'related', 'this', 'that', 'their', + 'national', 'federal', 'government', 'law', 'laws', 'program', 'programs', +]); + +/** Lower-case, strip punctuation/accents, drop short + stop words. Returns + * a de-duplicated token array. */ +export function tokenize(s) { + const raw = String(s ?? '') + .toLowerCase() + .normalize('NFKD') + .replace(/[̀-ͯ]/g, '') // strip combining diacritics after NFKD (e.g. é -> e) + .replace(/[^a-z0-9\s]/g, ' ') + .split(/\s+/) + .filter((t) => t.length >= 4 && !STOPWORDS.has(t)); + return Array.from(new Set(raw)); +} + +const slugOfBill = (b) => `${b.bill_type}-${b.bill_number}-${b.congress_number}`.toLowerCase(); + +/** Build the free-match index once per run: slug -> token set drawn from + * the bill's title + press_names (data/bills.json fields; there is no + * separate search-inputs.json). Bills with no usable tokens are skipped. */ +export function buildBillIndex(bills) { + const index = []; + for (const b of bills) { + const text = [b.title, ...((b.press_names ?? []))].filter(Boolean).join(' '); + const tokens = new Set(tokenize(text)); + if (tokens.size === 0) continue; + index.push({ slug: slugOfBill(b), title: b.title, tokens }); + } + return index; +} + +const T2_CONFIDENT_MIN_SHARED = 3; +const T2_CONFIDENT_MIN_RATIO = 0.6; +const T2_CANDIDATE_MIN_SHARED = 2; + +/** Score every indexed bill against a headline's tokens by raw shared-token + * count (+ratio of the headline's own tokens). Sorted best-first. */ +export function scoreCandidates(headline, billIndex) { + const hTokens = tokenize(headline); + if (hTokens.length === 0) return []; + const scored = []; + for (const entry of billIndex) { + let shared = 0; + for (const t of hTokens) if (entry.tokens.has(t)) shared++; + if (shared >= T2_CANDIDATE_MIN_SHARED) { + scored.push({ slug: entry.slug, title: entry.title, shared, ratio: shared / hTokens.length }); + } + } + scored.sort((a, b) => b.shared - a.shared || b.ratio - a.ratio); + return scored; +} + +/** t2 verdict for one headline against the index: + * { tier: 't2', slug } - one candidate is clearly the best match + * { tier: 'ambiguous', candidates } - 1+ plausible candidates, none + * clearly separated - t3's job + * null - no local signal at all */ +export function matchLocal(headline, billIndex) { + const candidates = scoreCandidates(headline, billIndex); + if (candidates.length === 0) return null; + const [top, runnerUp] = candidates; + const confident = + top.shared >= T2_CONFIDENT_MIN_SHARED && + top.ratio >= T2_CONFIDENT_MIN_RATIO && + (!runnerUp || top.shared >= runnerUp.shared * 1.5); + if (confident) return { tier: 't2', slug: top.slug }; + return { tier: 'ambiguous', candidates: candidates.slice(0, 5) }; +} + +// ---- t3 gating: only headlines that look legislative ------------------- +const LEGISLATIVE_SIGNAL_RE = /\b(bill|act|legislation|resolution|congress|senate|house|vote|voted|passed|introduced|amendment|committee|markup|filibuster|cloture|veto|vetoed|lawmakers?|representatives?|senators?)\b/i; + +/** Cheap pre-filter: does this headline look like it MIGHT be about a + * specific bill, before spending an LLM call disambiguating it? */ +export function looksLegislative(headline) { + return LEGISLATIVE_SIGNAL_RE.test(String(headline ?? '')); +} + +// ---- the ≥2-outlet rule ------------------------------------------------- +/** + * Decide which bills fire this run. + * citationSlugs: Set matched by an explicit citation (t1) this run + * - fires on any single outlet, no corroboration needed. + * outletsBySlug: Map> - outlets that matched this slug + * via t2/t3, ACCUMULATED across runs (the caller persists this in the + * seen-headlines cache) so corroboration can build up over multiple + * hourly polls, not just within one run's fetch window. + * Returns { fired: Set, reason: Map }. + */ +export function decideFires(citationSlugs, outletsBySlug) { + const fired = new Set(); + const reason = new Map(); + for (const slug of citationSlugs) { + fired.add(slug); + reason.set(slug, 'citation'); + } + for (const [slug, outlets] of outletsBySlug) { + if (fired.has(slug)) continue; + if (outlets && outlets.size >= 2) { + fired.add(slug); + reason.set(slug, 'corroborated'); + } + } + return { fired, reason }; +} + +// ---- dedupe cache keys --------------------------------------------------- +export function normalizeHeadlineKey(title, outlet) { + return `${String(title ?? '').toLowerCase().replace(/\s+/g, ' ').trim()}::${String(outlet ?? '').toLowerCase()}`; +} + +/** Stable hash of a (title, outlet) pair for the seen-headlines cache. */ +export function hashHeadline(title, outlet) { + return createHash('sha1').update(normalizeHeadlineKey(title, outlet)).digest('hex'); +} + +// ---- the no-change-no-commit guard -------------------------------------- +/** Given the syncOneBill outcome strings from this run's ON-FIRE actions, + * did anything actually mutate bills/es? Only 'refreshed' and 'added' + * touch the in-memory corpus; 'budget' (decode cap hit) and 'failed' + * don't. newsdesk.mjs only calls writeFileSync when this is true, so an + * hourly run with nothing to do never produces a diff for the workflow's + * own `git diff --cached --quiet` step to (redundantly, but harmlessly) + * confirm. */ +export function anyDataChanged(outcomes) { + return outcomes.some((o) => o === 'refreshed' || o === 'added'); +} + +// ---- RSS/Atom feed parsing (pure — takes already-fetched XML text) ------ +function decodeEntities(s) { + return String(s ?? '') + .replace(//g, '$1') + .replace(/&/g, '&') + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/&#(\d+);/g, (_, d) => String.fromCharCode(Number(d))) + .replace(/&#x([0-9a-f]+);/gi, (_, h) => String.fromCharCode(parseInt(h, 16))) + .trim(); +} + +function extractTag(block, tag) { + const m = block.match(new RegExp(`<${tag}(?:\\s[^>]*)?>([\\s\\S]*?)<\\/${tag}>`, 'i')); + return m ? decodeEntities(m[1]) : null; +} + +function extractLink(block) { + // Atom: ; RSS: https://... + const atom = block.match(/]*\bhref=["']([^"']+)["']/i); + if (atom) return atom[1]; + return extractTag(block, 'link'); +} + +/** Google News RSS (and some aggregator feeds) carry a per-article + * Outlet Name tag — use its + * domain when present so a single aggregator feed still yields correct + * per-article outlet attribution for the ≥2-outlet rule. */ +function extractSource(block) { + const m = block.match(/]*\burl=["']([^"']+)["'][^>]*>/i); + if (!m) return null; + try { + return new URL(m[1]).hostname.replace(/^www\./, ''); + } catch { + return null; + } +} + +/** Parse RSS 2.0 or Atom blocks out of raw feed XML into + * `{title, link, pubDate, source}[]`. Best-effort/regex-based (no XML + * dependency) — tolerant of the handful of real-world shapes the verified + * feed list actually returns (see newsdesk.mjs's SOURCES header comment). + * Entries missing a title or link are dropped. */ +export function parseFeed(xml) { + const blocks = [...String(xml ?? '').matchAll(/]*>([\s\S]*?)<\/item>|]*>([\s\S]*?)<\/entry>/gi)]; + const out = []; + for (const m of blocks) { + const body = m[1] ?? m[2] ?? ''; + const title = extractTag(body, 'title'); + const link = extractLink(body); + if (!title || !link) continue; + const pubDate = extractTag(body, 'pubDate') || extractTag(body, 'published') || extractTag(body, 'updated'); + out.push({ title, link, pubDate, source: extractSource(body) }); + } + return out; +} diff --git a/scripts/newsdesk.mjs b/scripts/newsdesk.mjs new file mode 100644 index 0000000..41c7aa9 --- /dev/null +++ b/scripts/newsdesk.mjs @@ -0,0 +1,333 @@ +/** + * Hourly headline-triggered bill resync (Part 2 of the 2026-07-16 + * spend-reduction pair; Part 1 is scripts/decode-gate.mjs + + * scripts/sync-bills.mjs). Owner directive: a news-headline trigger with + * ALL-IN cost under $5/day. + * + * node --env-file=.env.local scripts/newsdesk.mjs + * + * Needs CONGRESS_API_KEY + ANTHROPIC_API_KEY. + * + * ---- SOURCES: free RSS only, no paid APIs ---- + * NEWS_API_KEY / TheNewsAPI is deliberately NOT used here — that quota + * belongs to scripts/sync-coverage.mjs (which already exceeds its own + * daily quota some nights; pipeline-audit.md §4). Politically-balanced + * basket of 6 feeds (leans per data/media-bias.json), each verified live + * 2026-07-16 to return parseable RSS/Atom with real items: + * The Hill thehill.com center https://thehill.com/homenews/feed/ + * Roll Call rollcall.com unrated https://rollcall.com/feed/ + * (congress-focused trade pub, not AllSides-rated; + * included for direct legislative signal, not lean + * balance) + * NPR Politics npr.org center https://feeds.npr.org/1014/rss.xml + * Fox News foxnews.com right https://moxie.foxnews.com/google-publisher/politics.xml + * CBS News cbsnews.com left https://www.cbsnews.com/latest/rss/politics + * Google News (per-article) spans https://news.google.com/rss/search?q=congress%20bill%20when:1d&hl=en-US&gl=US&ceid=US:en + * many leans - each item carries a tag + * that resolves to a bare outlet domain, giving true + * per-article outlet attribution from one aggregator feed. + * Basket = 1 right + 1 left + 2 center + 1 unrated congress trade pub + 1 + * cross-outlet aggregator, so no single lean can structurally dominate + * which bills accumulate outlet corroboration (see the ≥2-outlet rule + * below). Dead/rejected candidates during verification: apnews.com/hub/ + * politics.rss and apnews.com/rss (both 404 — AP discontinued most public + * RSS), politico.com/rss/politics08.xml (403), feeds.washingtonpost.com/ + * rss/politics (200 but an empty/stub body). + * + * ---- Matching, cheapest first (full design in scripts/newsdesk-match.mjs) ---- + * t1 citation regex (free) -> t2 local token overlap against corpus + * titles+press_names (free) -> t3 ONE batched claude-haiku-4-5-20251001 + * call for headlines t2 leaves ambiguous, skipped entirely (zero API + * calls) when that batch is empty. + * + * ---- Trigger rule (nonpartisan guardrail, non-negotiable) ---- + * A bill fires only if (a) matched by an explicit citation from ANY + * outlet, or (b) matched by t2/t3 and corroborated by >=2 DISTINCT + * outlets, accumulated across runs via the seen-headlines cache + * (newsdesk-match.mjs's decideFires). See that module's header comment for + * why single-outlet triggering on a soft match would be a partisan-skew + * prioritization channel that data/media-bias.json's display-only lean + * normalization does nothing to prevent. + * + * ---- ON FIRE ---- + * Refresh the bill's status/last_action_date (free) via the SAME shared + * syncOneBill (scripts/bill-decode.mjs) sync-bills.mjs uses. If the bill + * is NOT already in the corpus — only possible for a t1 citation match; + * t2/t3 can only ever resolve to a bill already in data/bills.json, by + * construction — decode it via that same decode-before-publish path, + * force-bypassing the priority gate (the press trigger's own corroboration + * IS the worthiness signal here). Bounded by TWO caps: NEWSDESK_DECODE_CAP + * per run and NEWSDESK_DAILY_DECODE_CAP per UTC day (persisted in the + * cache file) — see "Cost ceiling" below for why both exist. + * + * ---- Dedupe (no hourly commits) ---- + * A seen-headlines cache (hash of normalized title+outlet) at + * .newsdesk-cache/seen.json persists across runs via actions/cache in + * newsdesk.yml — restored from the most recent previous run (a + * restore-key prefix match) and always saved under a fresh run-scoped key, + * so GitHub's own "evict caches unused for 7 days" policy ages out stale + * state automatically with no TTL bookkeeping needed here. The same file + * also carries `pendingOutlets` (the per-slug outlet sets the >=2-outlet + * rule accumulates across runs) and `dailyDecodes` (the cost ceiling). + * Cache miss (first run ever, an evicted cache, or a corrupt file) + * degrades gracefully to empty state in loadCache(): a bill that's already + * fresh just gets refreshed again (idempotent no-op), and an + * already-decoded bill is never re-decoded (bySlug.has(slug) governs that, + * not the cache). + * + * ---- Cost ceiling ---- + * Haiku (t3): most hourly runs' ambiguous batch is empty, so the LLM call + * is skipped entirely (resolveWithHaiku) at $0; on an active-news hour a + * ~20-40-headline batch at small prompts runs roughly $0.002-0.005/call. + * Expected ~$0.12/day summed across 24 runs on a newsy day — an upper + * estimate; many real days are lower. Trigger decodes (Sonnet 5, + * ~$0.07-0.15/bill, same model/cost as sync-bills.mjs): a typical day + * triggers 0 brand-new-bill decodes ($0, since a fired bill is almost + * always already in the corpus and only needs a free refresh); a busy day + * with 1-3 genuine new-bill triggers costs ~$0.07-0.45. The PER-RUN cap + * (NEWSDESK_DECODE_CAP=3) alone does not bound the DAILY total — 24 runs x + * 3 would allow up to ~$10.80/day in an implausible black-swan scenario — + * so NEWSDESK_DAILY_DECODE_CAP=10 is a second, code-enforced ceiling + * (~$0.70-1.50/day even then), keeping the documented "<$2/day" ceiling + * true by construction rather than aspirational. See the introducing PR's + * report for the full typical/busy/hard-ceiling cost table. + * + * ---- Boundaries ---- + * NEVER writes data/coverage.json — that stays scripts/sync-coverage.mjs's + * (TheNewsAPI, display-only enrichment of already-known bills). A future + * integration could have sync-coverage.mjs prioritize newsdesk-triggered + * slugs first in its own urgency-ordered nightly queue; out of scope here. + * Never touches data/sync-state.json's nightly cursor — same reasoning as + * scripts/hot-bills.mjs: a same-day refresh/trigger pass is not the + * nightly backlog scan's own progress signal. + */ +import Anthropic from '@anthropic-ai/sdk'; +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { loadJSON, syncOneBill } from './bill-decode.mjs'; +import { slugOf } from './congress-fetch.mjs'; +import { + anyDataChanged, + buildBillIndex, + decideFires, + findCitations, + hashHeadline, + looksLegislative, + matchLocal, + parseFeed, +} from './newsdesk-match.mjs'; + +const NEWSDESK_DECODE_CAP = Number(process.env.NEWSDESK_DECODE_CAP ?? 3); +const NEWSDESK_DAILY_DECODE_CAP = Number(process.env.NEWSDESK_DAILY_DECODE_CAP ?? 10); +const CACHE_DIR = process.env.NEWSDESK_CACHE_DIR ?? '.newsdesk-cache'; +const CACHE_FILE = `${CACHE_DIR}/seen.json`; +const T3_MAX_HEADLINES = Number(process.env.NEWSDESK_T3_MAX_HEADLINES ?? 40); +const T3_MODEL = 'claude-haiku-4-5-20251001'; +const USER_AGENT = 'Mozilla/5.0 (compatible; OravanNewsdesk/1.0; +https://oravan.org)'; + +// See the header comment for the full basket rationale + verification date. +const SOURCES = [ + { name: 'The Hill', domain: 'thehill.com', url: 'https://thehill.com/homenews/feed/' }, + { name: 'Roll Call', domain: 'rollcall.com', url: 'https://rollcall.com/feed/' }, + { name: 'NPR Politics', domain: 'npr.org', url: 'https://feeds.npr.org/1014/rss.xml' }, + { name: 'Fox News Politics', domain: 'foxnews.com', url: 'https://moxie.foxnews.com/google-publisher/politics.xml' }, + { name: 'CBS News Politics', domain: 'cbsnews.com', url: 'https://www.cbsnews.com/latest/rss/politics' }, + { name: 'Google News (congress bill query)', domain: null, url: 'https://news.google.com/rss/search?q=congress%20bill%20when:1d&hl=en-US&gl=US&ceid=US:en' }, +]; + +async function fetchFeed(src) { + const res = await fetch(src.url, { + signal: AbortSignal.timeout(20_000), + headers: { 'User-Agent': USER_AGENT }, + }); + if (!res.ok) throw new Error(`HTTP ${res.status}`); + const xml = await res.text(); + return parseFeed(xml).map((it) => ({ + title: it.title, + link: it.link, + pubDate: it.pubDate, + outlet: it.source ?? src.domain, + feedName: src.name, + })); +} + +function loadCache() { + try { + const raw = JSON.parse(readFileSync(CACHE_FILE, 'utf8')); + return { + seen: new Set(raw.seen ?? []), + pendingOutlets: raw.pendingOutlets ?? {}, // slug -> outlet domain[] + dailyDecodes: raw.dailyDecodes ?? null, // {date: 'YYYY-MM-DD', count} + }; + } catch { + // Cache miss (first run, evicted, or corrupt) - degrade gracefully. + // See the header comment: firing again on an already-handled bill is + // idempotent, so losing this state costs a little redundant work, not + // correctness. + return { seen: new Set(), pendingOutlets: {}, dailyDecodes: null }; + } +} + +function saveCache(cache) { + mkdirSync(CACHE_DIR, { recursive: true }); + writeFileSync(CACHE_FILE, JSON.stringify({ + seen: [...cache.seen], + pendingOutlets: cache.pendingOutlets, + dailyDecodes: cache.dailyDecodes, + })); +} + +/** ONE batched Haiku call resolving t2-ambiguous headlines against their + * own short candidate lists. Never trusts a slug the batch didn't offer - + * a hallucinated slug from the model can't enter the pipeline. */ +async function resolveWithHaiku(anthropic, batch) { + if (batch.length === 0) return new Map(); // skip t3 entirely - zero API calls + const prompt = batch + .map((b, i) => `${i}. HEADLINE: ${b.title}\n CANDIDATES: ${b.candidates.map((c) => `${c.slug} = ${c.title}`).join(' | ')}`) + .join('\n'); + let text; + try { + const msg = await anthropic.messages.create({ + model: T3_MODEL, + max_tokens: 1024, + messages: [{ role: 'user', content: `For each numbered headline below, decide which ONE candidate bill (if any) it is actually reporting on. Only pick a candidate if the headline is clearly about that specific bill's provisions, vote, or status — not just a similar general topic. If none fit, use null. + +${prompt} + +Output STRICT JSON only, an array like [{"i":0,"slug":"hr-1234-119"},{"i":1,"slug":null}] — no prose, no markdown fences, no other text.` }], + }); + text = msg.content[0]?.type === 'text' ? msg.content[0].text : ''; + } catch (e) { + console.error(`t3 Haiku call failed: ${e.message}`); + return new Map(); // degrade gracefully - no t3 matches this run + } + try { + const jsonText = text.trim().replace(/^```json?\s*/i, '').replace(/```\s*$/, ''); + const parsed = JSON.parse(jsonText); + const out = new Map(); + for (const row of parsed) { + if (row && typeof row.i === 'number' && typeof row.slug === 'string') { + const validOffer = batch[row.i]?.candidates.some((c) => c.slug === row.slug); + if (validOffer) out.set(row.i, row.slug); + } + } + return out; + } catch (e) { + console.error(`t3 JSON parse failed: ${e.message}`); + return new Map(); + } +} + +// ---- main ---- +const anthropic = new Anthropic({ maxRetries: 8 }); +const bills = loadJSON('data/bills.json'); +const es = loadJSON('data/bills-es.json'); +const bySlug = new Map(bills.map((b) => [slugOf(b), b])); +const billIndex = buildBillIndex(bills); +const cache = loadCache(); + +const todayUTC = new Date().toISOString().slice(0, 10); +if (!cache.dailyDecodes || cache.dailyDecodes.date !== todayUTC) { + cache.dailyDecodes = { date: todayUTC, count: 0 }; // roll over at UTC midnight +} + +console.log(`newsdesk: fetching ${SOURCES.length} feeds`); +const results = await Promise.allSettled(SOURCES.map(fetchFeed)); +const items = []; +results.forEach((r, i) => { + if (r.status === 'fulfilled') { + items.push(...r.value); + console.log(` ${SOURCES[i].name}: ${r.value.length} items`); + } else { + console.error(` ${SOURCES[i].name} FAILED: ${r.reason?.message ?? r.reason}`); + } +}); + +// Dedupe against the seen-headlines cache: skip anything already processed +// in a previous run (see the header comment's Dedupe section). +const newItems = items.filter((it) => !cache.seen.has(hashHeadline(it.title, it.outlet))); +console.log(`${items.length} headlines fetched, ${newItems.length} new (not previously seen)`); + +const citationSlugs = new Set(); +const t3Batch = []; +const t3Items = []; // parallel to t3Batch +const localOutletsBySlug = new Map(); // this run's t2/t3 outlet contributions, per slug + +for (const it of newItems) { + const citations = findCitations(it.title); + if (citations.length > 0) { + for (const c of citations) citationSlugs.add(c.slug); + continue; // citation tier wins outright - no need to also run t2/t3 + } + const local = matchLocal(it.title, billIndex); + if (local?.tier === 't2') { + if (!localOutletsBySlug.has(local.slug)) localOutletsBySlug.set(local.slug, new Set()); + localOutletsBySlug.get(local.slug).add(it.outlet ?? 'unknown'); + } else if (local?.tier === 'ambiguous' && looksLegislative(it.title) && t3Batch.length < T3_MAX_HEADLINES) { + t3Batch.push({ title: it.title, candidates: local.candidates }); + t3Items.push(it); + } + // else: no local signal at all, or not legislative-looking - dropped. + // t2/t3 can only ever resolve to a bill already in the corpus, by + // construction (see newsdesk-match.mjs's header comment), so a headline + // about a genuinely brand-new bill with no citation is unmatchable here. +} + +const t3Results = await resolveWithHaiku(anthropic, t3Batch); +console.log(`t3: ${t3Batch.length} headline(s) batched${t3Batch.length ? '' : ' (skipped - empty batch)'}, ${t3Results.size} resolved`); +for (const [i, slug] of t3Results) { + const it = t3Items[i]; + if (!localOutletsBySlug.has(slug)) localOutletsBySlug.set(slug, new Set()); + localOutletsBySlug.get(slug).add(it.outlet ?? 'unknown'); +} + +// Merge this run's t2/t3 outlet contributions into the persisted pending +// state, THEN decide fires - corroboration accumulates across runs rather +// than resetting hourly (decideFires's header comment has the reasoning). +for (const [slug, outlets] of localOutletsBySlug) { + const merged = new Set(cache.pendingOutlets[slug] ?? []); + for (const o of outlets) merged.add(o); + cache.pendingOutlets[slug] = [...merged]; +} +const pendingOutletsMap = new Map( + Object.entries(cache.pendingOutlets).map(([slug, outlets]) => [slug, new Set(outlets)]) +); +const { fired, reason } = decideFires(citationSlugs, pendingOutletsMap); +console.log(`fired this run: ${fired.size}${fired.size ? ' (' + [...fired].map((s) => `${s}:${reason.get(s)}`).join(', ') + ')' : ''}`); + +// ---- ON FIRE: refresh (free) or decode (gated by both caps) ---- +const forceSlugs = new Set(fired); // the press trigger's own corroboration stands in for the status gate +const outcomes = []; +let decodedThisRun = 0; +for (const slug of fired) { + const [type, number] = slug.split('-'); + const allowDecode = decodedThisRun < NEWSDESK_DECODE_CAP && cache.dailyDecodes.count < NEWSDESK_DAILY_DECODE_CAP; + const result = await syncOneBill({ type, number }, { allowDecode, forceSlugs, bills, es, bySlug, anthropic }); + outcomes.push(result.outcome); + if (result.outcome === 'added') { decodedThisRun++; cache.dailyDecodes.count++; } + if (result.outcome === 'refreshed' || result.outcome === 'added') { + delete cache.pendingOutlets[slug]; // corroboration spent - a future re-fire needs fresh corroboration + } + console.log(` ${slug}: ${result.outcome} (${reason.get(slug)})`); +} + +// ---- persist: cache always, data files only if something actually changed ---- +// Every headline this run touched (matched or not, fired or not) is marked +// seen so it isn't reprocessed next hour. The one accepted tradeoff: a +// citation-matched brand-new bill that hits BOTH decode caps this run +// ('budget' outcome) still gets its headline marked seen, so it won't +// retrigger from that exact article next hour - but a genuinely newsworthy +// bill almost always accumulates fresh headlines hour over hour, and even +// absent that, the nightly sync's own priority gate (scripts/decode-gate.mjs) +// will pick it up within a day once it has real recorded motion. +for (const it of newItems) cache.seen.add(hashHeadline(it.title, it.outlet)); +saveCache(cache); + +if (anyDataChanged(outcomes)) { + writeFileSync('data/bills.json', JSON.stringify(bills)); + writeFileSync('data/bills-es.json', JSON.stringify(es)); + const refreshedCount = outcomes.filter((o) => o === 'refreshed').length; + const addedCount = outcomes.filter((o) => o === 'added').length; + console.log(`DONE: ${refreshedCount} refreshed, ${addedCount} added+decoded; corpus ${bills.length}`); +} else { + console.log('DONE: no data changes this run - nothing written (the workflow commit step will no-op)'); +} diff --git a/tests/newsdesk-match.unit.spec.ts b/tests/newsdesk-match.unit.spec.ts new file mode 100644 index 0000000..b5d7246 --- /dev/null +++ b/tests/newsdesk-match.unit.spec.ts @@ -0,0 +1,215 @@ +import { expect, test } from '@playwright/test'; +// Pure, I/O-free module (no CONGRESS_API_KEY/ANTHROPIC_API_KEY, no network) +// - see scripts/newsdesk-match.mjs's header comment for the full match +// design this pins. +import { + anyDataChanged, + buildBillIndex, + decideFires, + findCitations, + hashHeadline, + looksLegislative, + matchLocal, + parseFeed, +} from '../scripts/newsdesk-match.mjs'; + +test.describe('findCitations (t1 explicit bill-number citations)', () => { + test('H.R. 1234 resolves to hr-1234-119', () => { + expect(findCitations('House passes H.R. 1234 in bipartisan vote')).toEqual([ + { type: 'hr', number: '1234', slug: 'hr-1234-119' }, + ]); + }); + + test('HR1234 (no punctuation/space) resolves the same way', () => { + expect(findCitations('Senate committee advances HR1234')).toEqual([ + { type: 'hr', number: '1234', slug: 'hr-1234-119' }, + ]); + }); + + test('S. 567 resolves to s-567-119', () => { + expect(findCitations('Lawmakers debate S. 567 funding measure')).toEqual([ + { type: 's', number: '567', slug: 's-567-119' }, + ]); + }); + + test('H. Res. 12 is NOT tracked - must not match (simple House resolution, not hr/s/hjres/sjres)', () => { + expect(findCitations('A new H. Res. 12 honors the local team')).toEqual([]); + }); + + test('"US 567" must not match S. - no word boundary inside "US"', () => { + expect(findCitations('US 567 highway expansion project moves forward')).toEqual([]); + }); + + test('H.J.Res. 45 and the bare HJRES form both resolve to hjres-45-119', () => { + expect(findCitations('H.J.Res. 45 disapproval resolution passes House')).toEqual([ + { type: 'hjres', number: '45', slug: 'hjres-45-119' }, + ]); + expect(findCitations('HJRES 45 clears procedural hurdle')).toEqual([ + { type: 'hjres', number: '45', slug: 'hjres-45-119' }, + ]); + }); + + test('S.J.Res. 9 and the glued SJRES9 form both resolve to sjres-9-119', () => { + expect(findCitations('S.J.Res. 9 heads to the floor')).toEqual([ + { type: 'sjres', number: '9', slug: 'sjres-9-119' }, + ]); + expect(findCitations('SJRES9 gets a vote')).toEqual([ + { type: 'sjres', number: '9', slug: 'sjres-9-119' }, + ]); + }); + + test('multiple distinct citations in one headline are all found, deduped', () => { + expect(findCitations('House passes H.R. 1234 while Senate weighs S. 45')).toEqual([ + { type: 'hr', number: '1234', slug: 'hr-1234-119' }, + { type: 's', number: '45', slug: 's-45-119' }, + ]); + }); + + test('case-insensitive', () => { + expect(findCitations('hr1234 trending on social media')).toEqual([ + { type: 'hr', number: '1234', slug: 'hr-1234-119' }, + ]); + }); + + test('no citation-shaped text yields an empty array', () => { + expect(findCitations('Local bakery wins county fair blue ribbon')).toEqual([]); + }); +}); + +const BILLS = [ + { bill_type: 'hr', bill_number: 8463, congress_number: 119, title: 'Prevent Government Fraud Act of 2026', press_names: ['SAVE Act'] }, + { bill_type: 's', bill_number: 180, congress_number: 119, title: 'Secondary Exposure Act', press_names: null }, + { bill_type: 'hr', bill_number: 99, congress_number: 119, title: 'A generic bill about roads and bridges', press_names: null }, +]; + +test.describe('matchLocal (t2 free token-overlap match)', () => { + const index = buildBillIndex(BILLS); + + test('a headline naming the press_name + title words confidently matches one bill', () => { + expect(matchLocal('Congress passes the SAVE Act to prevent government fraud', index)).toEqual({ + tier: 't2', + slug: 'hr-8463-119', + }); + }); + + test('a headline with weak, tied overlap across two bills is ambiguous (t3-bound), not a guess', () => { + const result = matchLocal('Secondary exposure concerns raised about bridges funding', index); + expect(result?.tier).toBe('ambiguous'); + if (!result || !('candidates' in result)) throw new Error('unreachable'); + expect((result.candidates ?? []).map((c: { slug: string }) => c.slug).sort()).toEqual(['hr-99-119', 's-180-119']); + }); + + test('a headline with no meaningful overlap matches nothing', () => { + expect(matchLocal('Local weather turns cooler this weekend', index)).toBeNull(); + }); +}); + +test.describe('looksLegislative (t3 batch gate)', () => { + test('legislative-signal headlines pass', () => { + expect(looksLegislative('Senate passes major infrastructure bill')).toBe(true); + expect(looksLegislative('Committee advances markup on tax measure')).toBe(true); + }); + + test('non-legislative headlines do not, keeping the Haiku batch small', () => { + expect(looksLegislative('Local bakery wins award')).toBe(false); + expect(looksLegislative('')).toBe(false); + }); +}); + +test.describe('decideFires (the >=2-outlet corroboration rule)', () => { + test('a citation match fires off a SINGLE outlet - no corroboration required', () => { + const { fired, reason } = decideFires(new Set(['hr-1-119']), new Map()); + expect(fired).toEqual(new Set(['hr-1-119'])); + expect(reason.get('hr-1-119')).toBe('citation'); + }); + + test('a t2/t3 match from exactly ONE outlet does NOT fire', () => { + const outlets = new Map([['s-2-119', new Set(['cbsnews.com'])]]); + const { fired } = decideFires(new Set(), outlets); + expect(fired.size).toBe(0); + }); + + test('a t2/t3 match from TWO distinct outlets fires as corroborated', () => { + const outlets = new Map([['s-3-119', new Set(['cbsnews.com', 'foxnews.com'])]]); + const { fired, reason } = decideFires(new Set(), outlets); + expect(fired).toEqual(new Set(['s-3-119'])); + expect(reason.get('s-3-119')).toBe('corroborated'); + }); + + test('the SAME outlet appearing twice does not count as two outlets (Set dedupes)', () => { + const outlets = new Map([['s-4-119', new Set(['cbsnews.com'])]]); // caller already deduped by Set + const { fired } = decideFires(new Set(), outlets); + expect(fired.size).toBe(0); + }); + + test('citation and corroboration combine without double-counting or colliding', () => { + const citations = new Set(['hr-1-119']); + const outlets = new Map([ + ['hr-1-119', new Set(['cbsnews.com'])], // also has a lone t2 hit - citation reason wins + ['s-3-119', new Set(['thehill.com', 'npr.org'])], + ]); + const { fired, reason } = decideFires(citations, outlets); + expect(fired).toEqual(new Set(['hr-1-119', 's-3-119'])); + expect(reason.get('hr-1-119')).toBe('citation'); + expect(reason.get('s-3-119')).toBe('corroborated'); + }); +}); + +test.describe('anyDataChanged (the no-change-no-commit guard)', () => { + test('no fired bills at all -> no change', () => { + expect(anyDataChanged([])).toBe(false); + }); + + test('every outcome deferred/failed -> no change (nothing to commit)', () => { + expect(anyDataChanged(['budget', 'failed', 'budget'])).toBe(false); + }); + + test('a free refresh alone counts as a change', () => { + expect(anyDataChanged(['refreshed'])).toBe(true); + }); + + test('a decode alone counts as a change', () => { + expect(anyDataChanged(['budget', 'added', 'failed'])).toBe(true); + }); +}); + +test.describe('hashHeadline (seen-headlines dedupe key)', () => { + test('normalizes whitespace and case so near-identical entries collide on purpose', () => { + expect(hashHeadline('Some Title', 'cbsnews.com')).toBe(hashHeadline('some title ', 'CBSNEWS.com')); + }); + + test('different outlets for the same title hash differently (per-outlet dedupe)', () => { + expect(hashHeadline('Some Title', 'cbsnews.com')).not.toBe(hashHeadline('Some Title', 'foxnews.com')); + }); +}); + +test.describe('parseFeed (RSS/Atom, pure string parsing)', () => { + test('parses an RSS 2.0 ', () => { + const xml = 'Test Headlinehttps://example.com/aThu, 16 Jul 2026 12:00:00 GMT'; + expect(parseFeed(xml)).toEqual([ + { title: 'Test Headline', link: 'https://example.com/a', pubDate: 'Thu, 16 Jul 2026 12:00:00 GMT', source: null }, + ]); + }); + + test('parses an Atom with href-style ', () => { + const xml = 'Atom Title2026-07-16T12:00:00Z'; + expect(parseFeed(xml)).toEqual([ + { title: 'Atom Title', link: 'https://example.com/b', pubDate: '2026-07-16T12:00:00Z', source: null }, + ]); + }); + + test('extracts the per-article outlet domain from a Google-News-style tag', () => { + const xml = 'Bipartisan Medicare Bill Unites Congresshttps://news.google.com/rss/articles/XLegis1'; + expect(parseFeed(xml)[0].source).toBe('legis1.com'); + }); + + test('drops entries missing a title or link', () => { + const xml = 'No link herehttps://example.com/c'; + expect(parseFeed(xml)).toEqual([]); + }); + + test('decodes CDATA and HTML entities in titles', () => { + const xml = '<![CDATA[Cruz & Democrats push back]]>https://example.com/d'; + expect(parseFeed(xml)[0].title).toBe('Cruz & Democrats push back'); + }); +}); From 3bac1ee235bea3d8357ddcbd824ac541bcc8b80c Mon Sep 17 00:00:00 2001 From: Colby Maxwell Date: Thu, 16 Jul 2026 13:50:02 -0400 Subject: [PATCH 3/3] fix(sync): map hyphenated 'Mark-up' action text to markup status 133 live corpus bills carry the hyphenated spelling and would have been wrongly gated out as mere 'committee'. Key check moves from import time to first cg() call so the module's pure exports stay unit-testable. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01XexCqeBQ2csGYiBMC62p2G --- scripts/congress-fetch.mjs | 15 +++++++++++++-- tests/decode-gate.unit.spec.ts | 9 +++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/scripts/congress-fetch.mjs b/scripts/congress-fetch.mjs index 62d9afb..03c148e 100644 --- a/scripts/congress-fetch.mjs +++ b/scripts/congress-fetch.mjs @@ -17,12 +17,16 @@ export const BILL_TYPES = new Set(['hr', 's', 'hjres', 'sjres']); const API = 'https://api.congress.gov/v3'; const KEY = process.env.CONGRESS_API_KEY; -if (!KEY) throw new Error('CONGRESS_API_KEY missing'); +// Key is checked at first fetch, not at import: this module also exports +// pure functions (mapStatus, urgencyScore, ...) that unit tests import +// without any secrets. Sync scripts still fail on their first cg() call +// with the same message. /** GET one Congress.gov endpoint, retrying on a bad status or a thrown/timed * out request (a hung socket must retry, not kill the whole run - the * 2026-06-13 crash). */ export async function cg(path, params = {}) { + if (!KEY) throw new Error('CONGRESS_API_KEY missing'); const url = new URL(`${API}${path}`); for (const [k, v] of Object.entries(params)) url.searchParams.set(k, v); url.searchParams.set('api_key', KEY); @@ -76,7 +80,14 @@ export function mapStatus(actionText) { text.includes('cloture') || text.includes('rule provid') || text.includes('motion to proceed') ) return 'floor_vote'; - if (text.includes('markup') || text.includes('ordered to be reported') || text.includes('reported by')) return 'markup'; + // 'mark-up': Congress.gov action text uses both spellings ("Mark-up + // Session Held") — the hyphenated form alone covers 133 live corpus bills + // that would otherwise read as mere 'committee' and be gated out of + // decoding (measured 2026-07-16; see scripts/decode-gate.mjs header). + if ( + text.includes('markup') || text.includes('mark-up') || + text.includes('ordered to be reported') || text.includes('reported by') + ) return 'markup'; return 'committee'; } diff --git a/tests/decode-gate.unit.spec.ts b/tests/decode-gate.unit.spec.ts index bbb4a18..1fd333e 100644 --- a/tests/decode-gate.unit.spec.ts +++ b/tests/decode-gate.unit.spec.ts @@ -3,6 +3,7 @@ import { expect, test } from '@playwright/test'; // scripts/decode-gate.mjs's header comment for the full status-distribution // numbers and the reasoning behind the chosen gate line. import { GATE_PASS_STATUSES, parseForceSlugs, passesGate } from '../scripts/decode-gate.mjs'; +import { mapStatus } from '../scripts/congress-fetch.mjs'; /* * These tests PIN the priority decode gate's status table. If a status here @@ -36,6 +37,14 @@ test.describe('passesGate (status table)', () => { ['conference', 'floor_vote', 'markup', 'passed_chamber', 'signed', 'vetoed'] ); }); + + test('hyphenated "Mark-up" action text maps to markup and clears the gate — 133 live bills depend on this spelling', () => { + expect(mapStatus('Committee Consideration and Mark-up Session Held')).toBe('markup'); + expect(mapStatus('Subcommittee Consideration and Markup Session Held')).toBe('markup'); + expect(passesGate(mapStatus('Committee Consideration and Mark-up Session Held'))).toBe(true); + // plain referral still reads as committee and stays gated + expect(mapStatus('Referred to the Committee on Energy and Commerce')).toBe('committee'); + }); }); test.describe('parseForceSlugs', () => {