diff --git a/.github/workflows/newsdesk.yml b/.github/workflows/newsdesk.yml new file mode 100644 index 0000000..968b208 --- /dev/null +++ b/.github/workflows/newsdesk.yml @@ -0,0 +1,119 @@ +name: Newsdesk headline trigger + +# Hourly headline-triggered bill resync (Part 2 of the 2026-07-16 +# spend-reduction pair; Part 1 is the priority decode gate in +# scripts/sync-bills.mjs). Polls a politically-balanced basket of free RSS +# feeds (NO paid news API - see scripts/newsdesk.mjs's header comment), +# matches headlines to bills cheapest-tier-first, and only acts on a bill +# that clears the >=2-distinct-outlet corroboration rule (or carries an +# explicit bill-number citation, which needs no corroboration). +# +# COST: expected ~$0.12/day Haiku (t3 disambiguation - most hourly runs' +# batch is empty, $0) + $0-0.45/day trigger decodes on a typical/busy day; +# NEWSDESK_DAILY_DECODE_CAP=10 (scripts/newsdesk.mjs) makes the documented +# <$2/day ceiling code-enforced, not just expected. Actions minutes: $0 +# (public repo, unlimited free minutes on standard runners). Full +# typical/busy/hard-ceiling breakdown in scripts/newsdesk.mjs's header +# comment and the introducing PR's report. + +on: + schedule: + - cron: '7 * * * *' # hourly, offset from the top of the hour + workflow_dispatch: + +permissions: + contents: write + +concurrency: + # Shared with sync-bills.yml/hot-bills.yml/refresh-legislators.yml so this + # can never race any of their commits to data/bills.json or data/bills-es.json. + group: data-sync + cancel-in-progress: false + +jobs: + newsdesk: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-node@v6 + # Needs npm ci (not stdlib-only like hot-bills.yml): scripts/newsdesk.mjs + # constructs its own Anthropic client for the t3 disambiguation call + # and the (shared, bill-decode.mjs) decode-before-publish path. + with: + node-version: 24 + cache: npm + - run: npm ci + - name: Restore seen-headlines cache + # Ever-growing-cache pattern (GitHub Actions caches can't be + # overwritten under the same key): restore-keys prefix-matches the + # most recent previous run's exact key; the save step below always + # writes a NEW run-scoped key. GitHub auto-evicts any cache entry + # unused for 7 days, which is exactly the "7-day retention" the spec + # asks for - no explicit TTL bookkeeping needed in the script itself. + # A cache miss (first run ever, or a fully-evicted history) degrades + # gracefully - see scripts/newsdesk.mjs's loadCache(). + uses: actions/cache/restore@v4 + with: + path: .newsdesk-cache + key: newsdesk-seen-${{ github.run_id }} + restore-keys: | + newsdesk-seen- + - name: Run newsdesk + env: + CONGRESS_API_KEY: ${{ secrets.CONGRESS_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + run: node scripts/newsdesk.mjs + - name: Save seen-headlines cache + # Always saves under this run's own unique key (see the restore + # step's comment) - if: always() so a mid-run failure still persists + # whatever cache state was written before the failure, rather than + # losing dedupe progress on every transient error. + if: always() + uses: actions/cache/save@v4 + with: + path: .newsdesk-cache + key: newsdesk-seen-${{ github.run_id }} + - name: Commit data + id: commit + # Same oravan-sync author + push-with-rebase-retry pattern as + # sync-bills.yml/hot-bills.yml - the author email must map to the + # Vercel-linked GitHub account or Vercel BLOCKs the auto-deploy + # (docs/solutions/vercel-bot-push-blocked-deploys.md). Commits + # ONLY when scripts/newsdesk.mjs actually changed data/ - an + # hourly no-op run (the common case) must never produce a deploy. + run: | + git config user.name "oravan-sync" + git config user.email "223600121+cm2489@users.noreply.github.com" + git add data/ + if git diff --cached --quiet; then + echo "no changes" + echo "changed=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + echo "changed=true" >> "$GITHUB_OUTPUT" + git commit -m "chore(data): newsdesk trigger $(date -u +%FT%HZ)" + # This job only ever touches data/bills.json/data/bills-es.json + # (no sync-state.json, no coverage.json - see scripts/newsdesk.mjs's + # Boundaries section) - disjoint from refresh-legislators.yml's + # files but overlapping sync-bills.yml's/hot-bills.yml's, all of + # which share this concurrency group. Rebase onto the latest main + # and retry, same pattern as those workflows. + for i in 1 2 3 4 5; do + if git push; then + echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + exit 0 + fi + echo "push rejected, rebasing onto latest main (attempt $i)" + git fetch origin main + git rebase origin/main + done + exit 1 + - name: Verify the deploy landed + # Same dead-man's-switch as sync-bills.yml/hot-bills.yml - poll + # production until it serves the SHA we just pushed. Skips with a + # notice until the PROD_URL repo variable is set. + if: steps.commit.outputs.changed == 'true' + env: + PROD_URL: ${{ vars.PROD_URL }} + EXPECT_SHA: ${{ steps.commit.outputs.sha }} + run: node scripts/verify-deploy.mjs diff --git a/.github/workflows/sync-bills.yml b/.github/workflows/sync-bills.yml index 6d96b3d..29fedcd 100644 --- a/.github/workflows/sync-bills.yml +++ b/.github/workflows/sync-bills.yml @@ -6,8 +6,11 @@ on: workflow_dispatch: inputs: max_new_decodes: - description: 'Max new bills to AI-decode this run' - default: '120' + description: 'Max new bills to AI-decode this run (safety ceiling — the priority gate does the real limiting)' + default: '60' + force_decode_slugs: + description: 'Comma-separated slugs to bypass the priority gate for (e.g. hr-1234-119,s-45-119)' + default: '' permissions: contents: write @@ -35,7 +38,8 @@ jobs: env: CONGRESS_API_KEY: ${{ secrets.CONGRESS_API_KEY }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - MAX_NEW_DECODES: ${{ inputs.max_new_decodes || '120' }} + MAX_NEW_DECODES: ${{ inputs.max_new_decodes || '60' }} + FORCE_DECODE_SLUGS: ${{ inputs.force_decode_slugs || '' }} run: node scripts/sync-bills.mjs - name: Backfill coverage search inputs # Drains press_names/news_query for bills decoded before search-input diff --git a/scripts/bill-decode.mjs b/scripts/bill-decode.mjs new file mode 100644 index 0000000..cae64ce --- /dev/null +++ b/scripts/bill-decode.mjs @@ -0,0 +1,249 @@ +/** + * Shared decode-before-publish + priority-gate resolution for ONE bill, + * used by BOTH scripts/sync-bills.mjs (nightly recent-first + ascending- + * backlog passes) and scripts/newsdesk.mjs (hourly headline-triggered + * resync, Part 2 of the 2026-07-16 spend-reduction pair). One copy so the + * gate, the FORCE_DECODE_SLUGS bypass, and the actual decode-before-publish + * AI calls can't drift between callers — same "one copy" discipline as + * lib/urgency.mjs's STATUS_BASE and congress-fetch.mjs's refreshBillFields. + * + * Extracted 2026-07-16 from what was previously sync-bills.mjs's own + * module-scope decode() + syncOneBill(): moving these here (as functions + * that take bills/es/bySlug/anthropic explicitly rather than closing over + * module-scope state) is what lets scripts/newsdesk.mjs decode a + * press-triggered new bill via the EXACT SAME decode-before-publish path + * the nightly sync uses, instead of maintaining a second copy of the + * summary/headline/ES prompts that could drift. + */ +import { readFileSync } from 'node:fs'; +import { + CONGRESS, + cg, + mapStatus, + refreshBillFields, + tagBill, + updateSlug, + urgencyScore, +} from './congress-fetch.mjs'; +import { passesGate } from './decode-gate.mjs'; +import { generateSearchInputs } from './search-inputs.mjs'; + +// Sonnet 5's tokenizer runs ~30% more tokens than 4.6 for the same text, so +// max_tokens caps on its calls are sized up accordingly; thinking is disabled +// explicitly because Sonnet 5 defaults it ON when the field is omitted, which +// would add unbounded thinking spend to batch calls. +export const DECODE_MODEL = 'claude-sonnet-5'; + +async function fetchBillText(type, number) { + const data = await cg(`/bill/${CONGRESS}/${type}/${number}/text`); + const versions = data.textVersions ?? []; + for (const v of [...versions].reverse()) { + const fmt = (v.formats ?? []).find((f) => f.type === 'Formatted Text'); + if (fmt?.url) { + const res = await fetch(fmt.url); + if (!res.ok) continue; + const html = await res.text(); + return html.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim().slice(0, 60_000); + } + } + return null; +} + +const DECODE_TAGS = [ + 'HEADLINE_EN', 'HEADLINE_ES', + 'TLDR', 'WHAT', 'WHO', 'WHY', 'COST', 'COST_CHIPS', + 'ES_TLDR', 'ES_WHAT', 'ES_WHO', 'ES_WHY', 'ES_COST', 'ES_COST_CHIPS', 'ES_SUMMARY', +]; + +function parseTagged(text) { + const out = {}; + for (let i = 0; i < DECODE_TAGS.length; i++) { + const tag = DECODE_TAGS[i]; + const start = text.indexOf(`[${tag}]`); + if (start === -1) throw new Error(`missing [${tag}]`); + const next = DECODE_TAGS.slice(i + 1) + .map((t) => text.indexOf(`[${t}]`)) + .filter((x) => x > start); + const end = next.length ? Math.min(...next) : text.length; + out[tag] = text.slice(start + tag.length + 2, end).trim(); + } + return out; +} + +const normCost = (s) => (s === 'NONE' || !s ? null : s); + +function normChips(s) { + if (s === 'NONE' || !s) return null; + const chips = s.split('|').map((c) => c.trim()).filter(Boolean); + if (chips.length < 1 || chips.length > 3 || chips.some((c) => c.length > 48)) return null; + return chips; +} + +async function decode(anthropic, bill, text) { + const sum = await anthropic.messages.create({ + model: DECODE_MODEL, max_tokens: 900, thinking: { type: 'disabled' }, + messages: [{ role: 'user', content: `Explain this congressional bill in plain language for an everyday US resident (8th-grade reading level). 2-3 short paragraphs: what it actually does, and who it affects. Strictly nonpartisan, no advocacy, no preamble, no markdown. + +Bill: ${bill.bill_type.toUpperCase()} ${bill.bill_number} — ${bill.title} + +Full text (may be truncated): +${text ?? bill.title}` }], + }); + const ai_summary = sum.content[0].text.trim(); + + const rest = await anthropic.messages.create({ + model: DECODE_MODEL, max_tokens: 3250, thinking: { type: 'disabled' }, + messages: [{ role: 'user', content: `From this plain-language bill summary, produce headlines, scannable sections, and a Spanish translation. + +Bill: ${bill.bill_type.toUpperCase()} ${bill.bill_number} +Summary: +${ai_summary} + +STRICT RULES: +- Use ONLY facts present in the summary. Never invent numbers, costs, or claims. +- Headlines: 45-90 chars, sentence case, factual news-desk style, varied construction (NOT "Topic — Consequence", avoid colons), never start with "Congress". Prioritize the most decision-relevant specifics: what it does, who it affects, what it costs, or where it stands. +- TLDR: one sentence, max 160 chars, the single most decision-relevant fact. +- WHAT: 1-3 sentences. WHO: 1-2. WHY: 1-2 sentences of neutral consequence, never benefits-framing. +- COST: 1-2 sentences ONLY if the summary contains spending/funding/fines/who-pays content; otherwise output exactly NONE (and ES_COST, COST_CHIPS, ES_COST_CHIPS all NONE too). +- COST_CHIPS: when COST exists, compress it to 2-3 chips separated by " | ", each a standalone fact fragment max 45 chars, sentence case, no period. Same count and order in ES_COST_CHIPS. If a fact can't fit 45 chars, output NONE for both chip tags (prose is the fallback). +- Spanish: natural Latin American Spanish, 8th-grade level; citations/numbers exact; agency names in English with a short gloss when helpful. ES_SUMMARY is the full summary translation. +- Plain text, no markdown. + +Output exactly this tagged format, each tag on its own line followed by its content: +[HEADLINE_EN] +[HEADLINE_ES] +[TLDR] +[WHAT] +[WHO] +[WHY] +[COST] +[COST_CHIPS] +[ES_TLDR] +[ES_WHAT] +[ES_WHO] +[ES_WHY] +[ES_COST] +[ES_COST_CHIPS] +[ES_SUMMARY]` }], + }); + const p = parseTagged(rest.content[0].text.trim()); + if (!p.HEADLINE_EN || !p.TLDR || !p.WHAT || !p.WHO || !p.WHY || !p.ES_SUMMARY) { + throw new Error('bad decode shape'); + } + return { + ai_summary, + ai_headline: p.HEADLINE_EN.slice(0, 110), + ai_sections: { + tldr: p.TLDR, what: p.WHAT, who: p.WHO, why: p.WHY, + cost: normCost(p.COST), costChips: normChips(p.COST_CHIPS), + }, + es_headline: p.HEADLINE_ES.slice(0, 110), + es_summary: p.ES_SUMMARY, + es_sections: { + tldr: p.ES_TLDR, what: p.ES_WHAT, who: p.ES_WHO, why: p.ES_WHY, + cost: normCost(p.ES_COST), costChips: normChips(p.ES_COST_CHIPS), + }, + }; +} + +/** + * Fetch one bill's current detail and either refresh it (already in the + * corpus — free, unconditional) or, for a brand-new bill, run it through + * the priority gate and decode-before-publish. The ONE place both + * sync-bills.mjs's passes and newsdesk.mjs's trigger path turn a + * Congress.gov update item ({type, number}) into a corpus mutation, so the + * gate, the force-bypass, and the refresh fields can't drift between + * callers. + * + * `u` is `{type, number}` (Congress.gov's shape, or newsdesk.mjs's own + * slug-derived equivalent). `ctx`: + * - allowDecode: this call may spend a decode if it clears the gate + * (the caller's own budget bookkeeping — MAX_NEW_DECODES for + * sync-bills.mjs, NEWSDESK_DECODE_CAP for newsdesk.mjs). + * - forceSlugs: a Set of slugs that bypass the priority gate entirely + * (still subject to allowDecode). Populated from FORCE_DECODE_SLUGS + * for manual/workflow_dispatch runs, or built in-process by + * newsdesk.mjs from headline-triggered bills — see decode-gate.mjs. + * - bills, es, bySlug, anthropic: the caller's loaded corpus + client. + * + * Returns one of: + * 'refreshed' — an existing bill's fields were updated in place (free) + * 'added' — a brand-new bill was decoded and pushed into the corpus + * 'gated' — a brand-new bill was found but shows no real legislative + * motion (and isn't force-bypassed) — NOT stored anywhere. + * Fully handled: if it later moves, Congress.gov's own + * updateDate advances past the caller's cursor and the + * update feed resurfaces it on a future run, when the gate + * re-evaluates against its then-current status. + * 'budget' — a brand-new bill cleared the gate (or was forced) but + * `allowDecode` was false this call + * 'failed' — the fetch or decode threw; `isNew` tells the caller + * whether this was a new-bill decode failure (must retry) + * or an existing bill's transient refresh failure + * (idempotent, self-heals on its next update). + */ +export async function syncOneBill(u, ctx) { + const { allowDecode, forceSlugs = new Set(), bills, es, bySlug, anthropic } = ctx; + const type = u.type.toLowerCase(); + const slug = updateSlug(u); + try { + const { bill: d } = await cg(`/bill/${CONGRESS}/${type}/${u.number}`); + const existing = bySlug.get(slug); + if (existing) { + refreshBillFields(existing, d); + return { outcome: 'refreshed', slug }; + } + const status = mapStatus(d.latestAction?.text); + const forced = forceSlugs.has(slug); + if (!forced && !passesGate(status)) { + return { outcome: 'gated', slug, status }; + } + if (!allowDecode) return { outcome: 'budget', slug }; + const lastActionDate = d.latestAction?.actionDate ?? null; + const bill = { + full_identifier: slug, + congress_number: CONGRESS, + bill_type: type, + bill_number: Number(u.number), + title: d.title, + short_title: null, + ai_summary: null, ai_headline: null, + sponsor_bioguide_id: d.sponsors?.[0]?.bioguideId ?? null, + introduced_date: d.introducedDate ?? null, + last_action_date: lastActionDate, + last_action_text: d.latestAction?.text ?? null, + status, + issue_tags: tagBill(d.policyArea?.name), + policy_area: d.policyArea?.name ?? null, + urgency_score: urgencyScore(status, lastActionDate), + congress_gov_url: `https://www.congress.gov/bill/${CONGRESS}th-congress/${type === 'hr' ? 'house-bill' : type === 's' ? 'senate-bill' : type === 'hjres' ? 'house-joint-resolution' : 'senate-joint-resolution'}/${u.number}`, + }; + const text = await fetchBillText(type, u.number); + const dec = await decode(anthropic, bill, text); + bill.ai_summary = dec.ai_summary; + bill.ai_headline = dec.ai_headline; + bill.ai_sections = dec.ai_sections; + // Search handles for the coverage sync (press names + subject query). + // Non-fatal: the backfill script sweeps up any misses. + try { + const si = await generateSearchInputs(anthropic, bill); + bill.press_names = si.press_names; + bill.news_query = si.news_query; + } catch (e) { + console.error(` search-inputs failed for ${slug}: ${e.message}`); + } + es[slug] = { headline: dec.es_headline, summary: dec.es_summary, sections: dec.es_sections }; + bills.push(bill); + bySlug.set(slug, bill); + return { outcome: 'added', slug }; + } catch (e) { + console.error(`FAIL ${slug}: ${e.message}`); + return { outcome: 'failed', slug, isNew: !bySlug.has(slug) }; + } +} + +/** Read+parse a data/*.json file — tiny shared helper so both callers open + * the corpus the same way. */ +export function loadJSON(path) { + return JSON.parse(readFileSync(path, 'utf8')); +} diff --git a/scripts/congress-fetch.mjs b/scripts/congress-fetch.mjs index 62d9afb..03c148e 100644 --- a/scripts/congress-fetch.mjs +++ b/scripts/congress-fetch.mjs @@ -17,12 +17,16 @@ export const BILL_TYPES = new Set(['hr', 's', 'hjres', 'sjres']); const API = 'https://api.congress.gov/v3'; const KEY = process.env.CONGRESS_API_KEY; -if (!KEY) throw new Error('CONGRESS_API_KEY missing'); +// Key is checked at first fetch, not at import: this module also exports +// pure functions (mapStatus, urgencyScore, ...) that unit tests import +// without any secrets. Sync scripts still fail on their first cg() call +// with the same message. /** GET one Congress.gov endpoint, retrying on a bad status or a thrown/timed * out request (a hung socket must retry, not kill the whole run - the * 2026-06-13 crash). */ export async function cg(path, params = {}) { + if (!KEY) throw new Error('CONGRESS_API_KEY missing'); const url = new URL(`${API}${path}`); for (const [k, v] of Object.entries(params)) url.searchParams.set(k, v); url.searchParams.set('api_key', KEY); @@ -76,7 +80,14 @@ export function mapStatus(actionText) { text.includes('cloture') || text.includes('rule provid') || text.includes('motion to proceed') ) return 'floor_vote'; - if (text.includes('markup') || text.includes('ordered to be reported') || text.includes('reported by')) return 'markup'; + // 'mark-up': Congress.gov action text uses both spellings ("Mark-up + // Session Held") — the hyphenated form alone covers 133 live corpus bills + // that would otherwise read as mere 'committee' and be gated out of + // decoding (measured 2026-07-16; see scripts/decode-gate.mjs header). + if ( + text.includes('markup') || text.includes('mark-up') || + text.includes('ordered to be reported') || text.includes('reported by') + ) return 'markup'; return 'committee'; } diff --git a/scripts/decode-gate.mjs b/scripts/decode-gate.mjs new file mode 100644 index 0000000..8e7a74e --- /dev/null +++ b/scripts/decode-gate.mjs @@ -0,0 +1,81 @@ +/** + * The priority decode gate (2026-07-16, owner directive: reduce spend right + * now by focusing the AI decode pass on a priority set of legislation — + * "what's up for a vote, what's in the news… the majority of the 2,147 + * bills is junk with high odds of never going anywhere"). Split into its + * own tiny, I/O-free module so the gate decision itself is directly + * unit-testable without mocking Congress.gov or Anthropic — see + * tests/decode-gate.unit.spec.ts. + * + * ---- Status distribution across the full 2,147-bill corpus, sampled + * 2026-07-16 (data/bills.json, mapStatus's output field — see + * scripts/congress-fetch.mjs) ---- + * committee: 1,706 (79.5%) + * floor_vote: 152 ( 7.1%) + * passed_chamber: 147 ( 6.8%) + * markup: 115 ( 5.4%) + * signed: 27 ( 1.3%) + * (conference, vetoed: 0 today — mapStatus supports both, neither is + * currently mapped onto any bill in the corpus) + * + * ---- The CRITICAL NUANCE: does 'committee' mean mere day-1 referral, or + * real committee action? ---- + * Sampled the 1,706 'committee'-status bills' last_action_text directly: + * 1,573/1,706 (92.2%) literally start with "Referred to the … Committee + * on …" — the automatic first action every single bill gets on + * introduction, zero legislative motion. The remaining 133 (7.8%) are + * genuine sub-committee activity (e.g. "Committee on Veterans' Affairs. + * Hearings held.", "Subcommittee Hearings Held", "Committee Consideration + * and Mark-up Session Held") that `mapStatus` happens to miscategorize as + * 'committee' instead of 'markup' — e.g. "Committee Consideration and + * Mark-up Session Held" doesn't match `text.includes('markup')` because of + * the hyphen in "Mark-up". `mapStatus` itself (scripts/congress-fetch.mjs) + * is out of scope for this change — this gate treats its 'committee' + * output as untrustworthy en masse rather than adding a second classifier + * here. + * + * ---- Chosen gate line ---- + * A bill passes ONLY if mapStatus returned something OTHER than + * 'committee' — markup / floor_vote / passed_chamber / conference / + * signed / vetoed all count as "real legislative motion" per the owner's + * directive; 'committee' does not, because it is dominated (92%) by mere + * referral. This intentionally gates out the 133 miscategorized + * real-action bills along with the 1,573 referral-only ones, trading a + * small amount of recall for a bright, cheap-to-reason-about line that + * needs no new classification surface. If that recall loss turns out to + * matter in practice, fixing the "Mark-up" hyphen gap in `mapStatus` is + * the correct follow-up, not a second condition here. + * + * At today's distribution, roughly 20.5% of bills (441/2,147) would clear + * this line — matching the owner's own framing that "the majority… is + * junk with high odds of never going anywhere." + */ + +export const GATE_PASS_STATUSES = new Set([ + 'markup', + 'floor_vote', + 'passed_chamber', + 'conference', + 'signed', + 'vetoed', +]); + +/** True if `status` (mapStatus's output) shows real legislative motion and + * should be allowed through the decode gate. */ +export function passesGate(status) { + return GATE_PASS_STATUSES.has(status); +} + +/** Parse a comma-separated slug list (FORCE_DECODE_SLUGS env, or a + * programmatically-built set) into a lower-cased Set. Bypasses the gate + * for exactly these slugs — used by workflow_dispatch manual runs and by + * scripts/newsdesk.mjs, which builds its own force set in-process from + * headline-matched bills rather than round-tripping through the env var. */ +export function parseForceSlugs(raw) { + return new Set( + String(raw ?? '') + .split(',') + .map((s) => s.trim().toLowerCase()) + .filter(Boolean) + ); +} diff --git a/scripts/newsdesk-match.mjs b/scripts/newsdesk-match.mjs new file mode 100644 index 0000000..52a71e5 --- /dev/null +++ b/scripts/newsdesk-match.mjs @@ -0,0 +1,287 @@ +/** + * Pure headline<->bill matching logic for scripts/newsdesk.mjs (Part 2 of + * the 2026-07-16 spend-reduction pair). Deliberately has ZERO imports of + * congress-fetch.mjs (which throws at import time without CONGRESS_API_KEY + * set) or '@anthropic-ai/sdk' — every function here is a plain string/data + * transform, so tests/newsdesk-match.unit.spec.ts can exercise the whole + * matching design (citation regex, local token overlap, the ≥2-outlet + * corroboration rule, feed parsing, the no-change-no-commit guard) with + * zero mocking and zero live network/API calls. + * + * ---- Three-tier match design, cheapest first ---- + * t1 (findCitations): regex over explicit bill-number citations + * ("H.R. 1234", "S. 567", "H.J.Res. 45"). Free, resolves directly to a + * slug for any of our four tracked types (hr, s, hjres, sjres; the + * 119th Congress). Fires on ANY single outlet — an explicit citation is + * unambiguous, so no corroboration is required. + * t2 (matchLocal): free normalized-token overlap against the corpus's own + * bill titles + press_names (data/bills.json fields — there is no + * separate data/search-inputs.json; press_names/news_query live + * directly on each bill object). A confident single match skips the LLM + * entirely; an ambiguous shortlist (multiple plausible candidates, or a + * weak-but-present signal on a legislative-looking headline) is handed + * to t3. + * t3 (resolved by scripts/newsdesk.mjs's one batched Haiku call): ONLY + * headlines t2 left ambiguous. This module supplies the batch + * membership test (looksLegislative) and validates the LLM's output + * against the offered candidates (a hallucinated slug is never trusted + * — see newsdesk.mjs's resolveWithHaiku). + * + * ---- The ≥2-outlet corroboration rule (decideFires) ---- + * A bill fires only if (a) an explicit citation matched it from ANY + * outlet, or (b) t2/t3 matched it from at least 2 DISTINCT outlets. (a) + * needs no corroboration because a citation is unambiguous. (b) does, + * because a free-text/LLM match to a bill's title is inherently softer, + * and — the nonpartisan guardrail this exists for — letting a single + * outlet's coverage alone decide which bills get fast-tracked ahead of + * others would make whichever outlet happens to publish first a de facto + * prioritization channel. data/media-bias.json's AllSides lean data + * already normalizes DISPLAY of an outlet's lean; it does nothing to stop + * a single-source story from silently jumping a bill to the front of the + * decode/refresh queue. Requiring 2 distinct outlets before a soft match + * can trigger anything makes that channel much harder to game with one + * placement, without blocking a bill that's genuinely breaking (which + * will show up in the citation tier, or in >1 outlet's feed within the + * same rolling window, almost immediately). + */ +import { createHash } from 'node:crypto'; + +// Duplicated from congress-fetch.mjs's CONGRESS constant (not imported) so +// this module stays import-clean for unit tests — see the header comment. +// The 119th Congress; bump alongside congress-fetch.mjs's own CONGRESS if +// the tracked Congress ever changes. +const CONGRESS = 119; +export const TRACKED_TYPES = new Set(['hr', 's', 'hjres', 'sjres']); + +// ---- t1: explicit bill-number citations ------------------------------ +// Each alternative requires the number to be IMMEDIATELY adjacent (through +// only an optional period and a single optional space) to the type token, +// which is what correctly rejects "H. Res. 12" (a simple House resolution +// — NOT one of our 4 tracked types; "Res" inserts non-dot/space characters +// between "H"/"R" and the digits, so no alternative can complete a match) +// and "US 567" (the leading \b can't fire inside "US" — no word boundary +// between "U" and "S"). HJRES/SJRES are tried before the shorter HR/S +// alternatives at each scan position so "H.J.Res. 45" resolves as hjres, +// not as a stray "H." partial. +const CITATION_RE = /\b(H\.?\s?J\.?\s?Res\.?|S\.?\s?J\.?\s?Res\.?|H\.?\s?R\.?|S\.?)\s?(\d{1,5})\b/gi; + +function normalizeType(raw) { + return raw.replace(/[^a-zA-Z]/g, '').toLowerCase(); +} + +const TYPE_ALIASES = { h: null, hr: 'hr', s: 's', hjres: 'hjres', sjres: 'sjres' }; + +/** Find every explicit, trackable bill-number citation in `text`. Returns + * `[{type, number, slug}]` — type is one of hr/s/hjres/sjres, slug is + * `${type}-${number}-119`. Citations to untracked types (e.g. "H. Res. + * 12", a simple resolution) are silently excluded, not returned as a + * partial/wrong match. */ +export function findCitations(text) { + const out = []; + const seen = new Set(); + for (const m of String(text ?? '').matchAll(CITATION_RE)) { + const type = TYPE_ALIASES[normalizeType(m[1])]; + if (!type) continue; + const number = String(Number(m[2])); // normalize away leading zeros, if any + const slug = `${type}-${number}-${CONGRESS}`; + if (seen.has(slug)) continue; + seen.add(slug); + out.push({ type, number, slug }); + } + return out; +} + +// ---- t2: free local token-overlap match ------------------------------- +const STOPWORDS = new Set([ + 'the', 'a', 'an', 'of', 'to', 'for', 'and', 'or', 'in', 'on', 'at', 'by', + 'with', 'from', 'into', 'act', 'acts', 'bill', 'bills', 'amendment', + 'amendments', 'congress', 'congressional', 'united', 'states', 'american', + 'establish', 'establishing', 'establishment', 'require', 'requiring', + 'provide', 'providing', 'relating', 'related', 'this', 'that', 'their', + 'national', 'federal', 'government', 'law', 'laws', 'program', 'programs', +]); + +/** Lower-case, strip punctuation/accents, drop short + stop words. Returns + * a de-duplicated token array. */ +export function tokenize(s) { + const raw = String(s ?? '') + .toLowerCase() + .normalize('NFKD') + .replace(/[̀-ͯ]/g, '') // strip combining diacritics after NFKD (e.g. é -> e) + .replace(/[^a-z0-9\s]/g, ' ') + .split(/\s+/) + .filter((t) => t.length >= 4 && !STOPWORDS.has(t)); + return Array.from(new Set(raw)); +} + +const slugOfBill = (b) => `${b.bill_type}-${b.bill_number}-${b.congress_number}`.toLowerCase(); + +/** Build the free-match index once per run: slug -> token set drawn from + * the bill's title + press_names (data/bills.json fields; there is no + * separate search-inputs.json). Bills with no usable tokens are skipped. */ +export function buildBillIndex(bills) { + const index = []; + for (const b of bills) { + const text = [b.title, ...((b.press_names ?? []))].filter(Boolean).join(' '); + const tokens = new Set(tokenize(text)); + if (tokens.size === 0) continue; + index.push({ slug: slugOfBill(b), title: b.title, tokens }); + } + return index; +} + +const T2_CONFIDENT_MIN_SHARED = 3; +const T2_CONFIDENT_MIN_RATIO = 0.6; +const T2_CANDIDATE_MIN_SHARED = 2; + +/** Score every indexed bill against a headline's tokens by raw shared-token + * count (+ratio of the headline's own tokens). Sorted best-first. */ +export function scoreCandidates(headline, billIndex) { + const hTokens = tokenize(headline); + if (hTokens.length === 0) return []; + const scored = []; + for (const entry of billIndex) { + let shared = 0; + for (const t of hTokens) if (entry.tokens.has(t)) shared++; + if (shared >= T2_CANDIDATE_MIN_SHARED) { + scored.push({ slug: entry.slug, title: entry.title, shared, ratio: shared / hTokens.length }); + } + } + scored.sort((a, b) => b.shared - a.shared || b.ratio - a.ratio); + return scored; +} + +/** t2 verdict for one headline against the index: + * { tier: 't2', slug } - one candidate is clearly the best match + * { tier: 'ambiguous', candidates } - 1+ plausible candidates, none + * clearly separated - t3's job + * null - no local signal at all */ +export function matchLocal(headline, billIndex) { + const candidates = scoreCandidates(headline, billIndex); + if (candidates.length === 0) return null; + const [top, runnerUp] = candidates; + const confident = + top.shared >= T2_CONFIDENT_MIN_SHARED && + top.ratio >= T2_CONFIDENT_MIN_RATIO && + (!runnerUp || top.shared >= runnerUp.shared * 1.5); + if (confident) return { tier: 't2', slug: top.slug }; + return { tier: 'ambiguous', candidates: candidates.slice(0, 5) }; +} + +// ---- t3 gating: only headlines that look legislative ------------------- +const LEGISLATIVE_SIGNAL_RE = /\b(bill|act|legislation|resolution|congress|senate|house|vote|voted|passed|introduced|amendment|committee|markup|filibuster|cloture|veto|vetoed|lawmakers?|representatives?|senators?)\b/i; + +/** Cheap pre-filter: does this headline look like it MIGHT be about a + * specific bill, before spending an LLM call disambiguating it? */ +export function looksLegislative(headline) { + return LEGISLATIVE_SIGNAL_RE.test(String(headline ?? '')); +} + +// ---- the ≥2-outlet rule ------------------------------------------------- +/** + * Decide which bills fire this run. + * citationSlugs: Set matched by an explicit citation (t1) this run + * - fires on any single outlet, no corroboration needed. + * outletsBySlug: Map> - outlets that matched this slug + * via t2/t3, ACCUMULATED across runs (the caller persists this in the + * seen-headlines cache) so corroboration can build up over multiple + * hourly polls, not just within one run's fetch window. + * Returns { fired: Set, reason: Map }. + */ +export function decideFires(citationSlugs, outletsBySlug) { + const fired = new Set(); + const reason = new Map(); + for (const slug of citationSlugs) { + fired.add(slug); + reason.set(slug, 'citation'); + } + for (const [slug, outlets] of outletsBySlug) { + if (fired.has(slug)) continue; + if (outlets && outlets.size >= 2) { + fired.add(slug); + reason.set(slug, 'corroborated'); + } + } + return { fired, reason }; +} + +// ---- dedupe cache keys --------------------------------------------------- +export function normalizeHeadlineKey(title, outlet) { + return `${String(title ?? '').toLowerCase().replace(/\s+/g, ' ').trim()}::${String(outlet ?? '').toLowerCase()}`; +} + +/** Stable hash of a (title, outlet) pair for the seen-headlines cache. */ +export function hashHeadline(title, outlet) { + return createHash('sha1').update(normalizeHeadlineKey(title, outlet)).digest('hex'); +} + +// ---- the no-change-no-commit guard -------------------------------------- +/** Given the syncOneBill outcome strings from this run's ON-FIRE actions, + * did anything actually mutate bills/es? Only 'refreshed' and 'added' + * touch the in-memory corpus; 'budget' (decode cap hit) and 'failed' + * don't. newsdesk.mjs only calls writeFileSync when this is true, so an + * hourly run with nothing to do never produces a diff for the workflow's + * own `git diff --cached --quiet` step to (redundantly, but harmlessly) + * confirm. */ +export function anyDataChanged(outcomes) { + return outcomes.some((o) => o === 'refreshed' || o === 'added'); +} + +// ---- RSS/Atom feed parsing (pure — takes already-fetched XML text) ------ +function decodeEntities(s) { + return String(s ?? '') + .replace(//g, '$1') + .replace(/&/g, '&') + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/&#(\d+);/g, (_, d) => String.fromCharCode(Number(d))) + .replace(/&#x([0-9a-f]+);/gi, (_, h) => String.fromCharCode(parseInt(h, 16))) + .trim(); +} + +function extractTag(block, tag) { + const m = block.match(new RegExp(`<${tag}(?:\\s[^>]*)?>([\\s\\S]*?)<\\/${tag}>`, 'i')); + return m ? decodeEntities(m[1]) : null; +} + +function extractLink(block) { + // Atom: ; RSS: https://... + const atom = block.match(/]*\bhref=["']([^"']+)["']/i); + if (atom) return atom[1]; + return extractTag(block, 'link'); +} + +/** Google News RSS (and some aggregator feeds) carry a per-article + * Outlet Name tag — use its + * domain when present so a single aggregator feed still yields correct + * per-article outlet attribution for the ≥2-outlet rule. */ +function extractSource(block) { + const m = block.match(/]*\burl=["']([^"']+)["'][^>]*>/i); + if (!m) return null; + try { + return new URL(m[1]).hostname.replace(/^www\./, ''); + } catch { + return null; + } +} + +/** Parse RSS 2.0 or Atom blocks out of raw feed XML into + * `{title, link, pubDate, source}[]`. Best-effort/regex-based (no XML + * dependency) — tolerant of the handful of real-world shapes the verified + * feed list actually returns (see newsdesk.mjs's SOURCES header comment). + * Entries missing a title or link are dropped. */ +export function parseFeed(xml) { + const blocks = [...String(xml ?? '').matchAll(/]*>([\s\S]*?)<\/item>|]*>([\s\S]*?)<\/entry>/gi)]; + const out = []; + for (const m of blocks) { + const body = m[1] ?? m[2] ?? ''; + const title = extractTag(body, 'title'); + const link = extractLink(body); + if (!title || !link) continue; + const pubDate = extractTag(body, 'pubDate') || extractTag(body, 'published') || extractTag(body, 'updated'); + out.push({ title, link, pubDate, source: extractSource(body) }); + } + return out; +} diff --git a/scripts/newsdesk.mjs b/scripts/newsdesk.mjs new file mode 100644 index 0000000..41c7aa9 --- /dev/null +++ b/scripts/newsdesk.mjs @@ -0,0 +1,333 @@ +/** + * Hourly headline-triggered bill resync (Part 2 of the 2026-07-16 + * spend-reduction pair; Part 1 is scripts/decode-gate.mjs + + * scripts/sync-bills.mjs). Owner directive: a news-headline trigger with + * ALL-IN cost under $5/day. + * + * node --env-file=.env.local scripts/newsdesk.mjs + * + * Needs CONGRESS_API_KEY + ANTHROPIC_API_KEY. + * + * ---- SOURCES: free RSS only, no paid APIs ---- + * NEWS_API_KEY / TheNewsAPI is deliberately NOT used here — that quota + * belongs to scripts/sync-coverage.mjs (which already exceeds its own + * daily quota some nights; pipeline-audit.md §4). Politically-balanced + * basket of 6 feeds (leans per data/media-bias.json), each verified live + * 2026-07-16 to return parseable RSS/Atom with real items: + * The Hill thehill.com center https://thehill.com/homenews/feed/ + * Roll Call rollcall.com unrated https://rollcall.com/feed/ + * (congress-focused trade pub, not AllSides-rated; + * included for direct legislative signal, not lean + * balance) + * NPR Politics npr.org center https://feeds.npr.org/1014/rss.xml + * Fox News foxnews.com right https://moxie.foxnews.com/google-publisher/politics.xml + * CBS News cbsnews.com left https://www.cbsnews.com/latest/rss/politics + * Google News (per-article) spans https://news.google.com/rss/search?q=congress%20bill%20when:1d&hl=en-US&gl=US&ceid=US:en + * many leans - each item carries a tag + * that resolves to a bare outlet domain, giving true + * per-article outlet attribution from one aggregator feed. + * Basket = 1 right + 1 left + 2 center + 1 unrated congress trade pub + 1 + * cross-outlet aggregator, so no single lean can structurally dominate + * which bills accumulate outlet corroboration (see the ≥2-outlet rule + * below). Dead/rejected candidates during verification: apnews.com/hub/ + * politics.rss and apnews.com/rss (both 404 — AP discontinued most public + * RSS), politico.com/rss/politics08.xml (403), feeds.washingtonpost.com/ + * rss/politics (200 but an empty/stub body). + * + * ---- Matching, cheapest first (full design in scripts/newsdesk-match.mjs) ---- + * t1 citation regex (free) -> t2 local token overlap against corpus + * titles+press_names (free) -> t3 ONE batched claude-haiku-4-5-20251001 + * call for headlines t2 leaves ambiguous, skipped entirely (zero API + * calls) when that batch is empty. + * + * ---- Trigger rule (nonpartisan guardrail, non-negotiable) ---- + * A bill fires only if (a) matched by an explicit citation from ANY + * outlet, or (b) matched by t2/t3 and corroborated by >=2 DISTINCT + * outlets, accumulated across runs via the seen-headlines cache + * (newsdesk-match.mjs's decideFires). See that module's header comment for + * why single-outlet triggering on a soft match would be a partisan-skew + * prioritization channel that data/media-bias.json's display-only lean + * normalization does nothing to prevent. + * + * ---- ON FIRE ---- + * Refresh the bill's status/last_action_date (free) via the SAME shared + * syncOneBill (scripts/bill-decode.mjs) sync-bills.mjs uses. If the bill + * is NOT already in the corpus — only possible for a t1 citation match; + * t2/t3 can only ever resolve to a bill already in data/bills.json, by + * construction — decode it via that same decode-before-publish path, + * force-bypassing the priority gate (the press trigger's own corroboration + * IS the worthiness signal here). Bounded by TWO caps: NEWSDESK_DECODE_CAP + * per run and NEWSDESK_DAILY_DECODE_CAP per UTC day (persisted in the + * cache file) — see "Cost ceiling" below for why both exist. + * + * ---- Dedupe (no hourly commits) ---- + * A seen-headlines cache (hash of normalized title+outlet) at + * .newsdesk-cache/seen.json persists across runs via actions/cache in + * newsdesk.yml — restored from the most recent previous run (a + * restore-key prefix match) and always saved under a fresh run-scoped key, + * so GitHub's own "evict caches unused for 7 days" policy ages out stale + * state automatically with no TTL bookkeeping needed here. The same file + * also carries `pendingOutlets` (the per-slug outlet sets the >=2-outlet + * rule accumulates across runs) and `dailyDecodes` (the cost ceiling). + * Cache miss (first run ever, an evicted cache, or a corrupt file) + * degrades gracefully to empty state in loadCache(): a bill that's already + * fresh just gets refreshed again (idempotent no-op), and an + * already-decoded bill is never re-decoded (bySlug.has(slug) governs that, + * not the cache). + * + * ---- Cost ceiling ---- + * Haiku (t3): most hourly runs' ambiguous batch is empty, so the LLM call + * is skipped entirely (resolveWithHaiku) at $0; on an active-news hour a + * ~20-40-headline batch at small prompts runs roughly $0.002-0.005/call. + * Expected ~$0.12/day summed across 24 runs on a newsy day — an upper + * estimate; many real days are lower. Trigger decodes (Sonnet 5, + * ~$0.07-0.15/bill, same model/cost as sync-bills.mjs): a typical day + * triggers 0 brand-new-bill decodes ($0, since a fired bill is almost + * always already in the corpus and only needs a free refresh); a busy day + * with 1-3 genuine new-bill triggers costs ~$0.07-0.45. The PER-RUN cap + * (NEWSDESK_DECODE_CAP=3) alone does not bound the DAILY total — 24 runs x + * 3 would allow up to ~$10.80/day in an implausible black-swan scenario — + * so NEWSDESK_DAILY_DECODE_CAP=10 is a second, code-enforced ceiling + * (~$0.70-1.50/day even then), keeping the documented "<$2/day" ceiling + * true by construction rather than aspirational. See the introducing PR's + * report for the full typical/busy/hard-ceiling cost table. + * + * ---- Boundaries ---- + * NEVER writes data/coverage.json — that stays scripts/sync-coverage.mjs's + * (TheNewsAPI, display-only enrichment of already-known bills). A future + * integration could have sync-coverage.mjs prioritize newsdesk-triggered + * slugs first in its own urgency-ordered nightly queue; out of scope here. + * Never touches data/sync-state.json's nightly cursor — same reasoning as + * scripts/hot-bills.mjs: a same-day refresh/trigger pass is not the + * nightly backlog scan's own progress signal. + */ +import Anthropic from '@anthropic-ai/sdk'; +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { loadJSON, syncOneBill } from './bill-decode.mjs'; +import { slugOf } from './congress-fetch.mjs'; +import { + anyDataChanged, + buildBillIndex, + decideFires, + findCitations, + hashHeadline, + looksLegislative, + matchLocal, + parseFeed, +} from './newsdesk-match.mjs'; + +const NEWSDESK_DECODE_CAP = Number(process.env.NEWSDESK_DECODE_CAP ?? 3); +const NEWSDESK_DAILY_DECODE_CAP = Number(process.env.NEWSDESK_DAILY_DECODE_CAP ?? 10); +const CACHE_DIR = process.env.NEWSDESK_CACHE_DIR ?? '.newsdesk-cache'; +const CACHE_FILE = `${CACHE_DIR}/seen.json`; +const T3_MAX_HEADLINES = Number(process.env.NEWSDESK_T3_MAX_HEADLINES ?? 40); +const T3_MODEL = 'claude-haiku-4-5-20251001'; +const USER_AGENT = 'Mozilla/5.0 (compatible; OravanNewsdesk/1.0; +https://oravan.org)'; + +// See the header comment for the full basket rationale + verification date. +const SOURCES = [ + { name: 'The Hill', domain: 'thehill.com', url: 'https://thehill.com/homenews/feed/' }, + { name: 'Roll Call', domain: 'rollcall.com', url: 'https://rollcall.com/feed/' }, + { name: 'NPR Politics', domain: 'npr.org', url: 'https://feeds.npr.org/1014/rss.xml' }, + { name: 'Fox News Politics', domain: 'foxnews.com', url: 'https://moxie.foxnews.com/google-publisher/politics.xml' }, + { name: 'CBS News Politics', domain: 'cbsnews.com', url: 'https://www.cbsnews.com/latest/rss/politics' }, + { name: 'Google News (congress bill query)', domain: null, url: 'https://news.google.com/rss/search?q=congress%20bill%20when:1d&hl=en-US&gl=US&ceid=US:en' }, +]; + +async function fetchFeed(src) { + const res = await fetch(src.url, { + signal: AbortSignal.timeout(20_000), + headers: { 'User-Agent': USER_AGENT }, + }); + if (!res.ok) throw new Error(`HTTP ${res.status}`); + const xml = await res.text(); + return parseFeed(xml).map((it) => ({ + title: it.title, + link: it.link, + pubDate: it.pubDate, + outlet: it.source ?? src.domain, + feedName: src.name, + })); +} + +function loadCache() { + try { + const raw = JSON.parse(readFileSync(CACHE_FILE, 'utf8')); + return { + seen: new Set(raw.seen ?? []), + pendingOutlets: raw.pendingOutlets ?? {}, // slug -> outlet domain[] + dailyDecodes: raw.dailyDecodes ?? null, // {date: 'YYYY-MM-DD', count} + }; + } catch { + // Cache miss (first run, evicted, or corrupt) - degrade gracefully. + // See the header comment: firing again on an already-handled bill is + // idempotent, so losing this state costs a little redundant work, not + // correctness. + return { seen: new Set(), pendingOutlets: {}, dailyDecodes: null }; + } +} + +function saveCache(cache) { + mkdirSync(CACHE_DIR, { recursive: true }); + writeFileSync(CACHE_FILE, JSON.stringify({ + seen: [...cache.seen], + pendingOutlets: cache.pendingOutlets, + dailyDecodes: cache.dailyDecodes, + })); +} + +/** ONE batched Haiku call resolving t2-ambiguous headlines against their + * own short candidate lists. Never trusts a slug the batch didn't offer - + * a hallucinated slug from the model can't enter the pipeline. */ +async function resolveWithHaiku(anthropic, batch) { + if (batch.length === 0) return new Map(); // skip t3 entirely - zero API calls + const prompt = batch + .map((b, i) => `${i}. HEADLINE: ${b.title}\n CANDIDATES: ${b.candidates.map((c) => `${c.slug} = ${c.title}`).join(' | ')}`) + .join('\n'); + let text; + try { + const msg = await anthropic.messages.create({ + model: T3_MODEL, + max_tokens: 1024, + messages: [{ role: 'user', content: `For each numbered headline below, decide which ONE candidate bill (if any) it is actually reporting on. Only pick a candidate if the headline is clearly about that specific bill's provisions, vote, or status — not just a similar general topic. If none fit, use null. + +${prompt} + +Output STRICT JSON only, an array like [{"i":0,"slug":"hr-1234-119"},{"i":1,"slug":null}] — no prose, no markdown fences, no other text.` }], + }); + text = msg.content[0]?.type === 'text' ? msg.content[0].text : ''; + } catch (e) { + console.error(`t3 Haiku call failed: ${e.message}`); + return new Map(); // degrade gracefully - no t3 matches this run + } + try { + const jsonText = text.trim().replace(/^```json?\s*/i, '').replace(/```\s*$/, ''); + const parsed = JSON.parse(jsonText); + const out = new Map(); + for (const row of parsed) { + if (row && typeof row.i === 'number' && typeof row.slug === 'string') { + const validOffer = batch[row.i]?.candidates.some((c) => c.slug === row.slug); + if (validOffer) out.set(row.i, row.slug); + } + } + return out; + } catch (e) { + console.error(`t3 JSON parse failed: ${e.message}`); + return new Map(); + } +} + +// ---- main ---- +const anthropic = new Anthropic({ maxRetries: 8 }); +const bills = loadJSON('data/bills.json'); +const es = loadJSON('data/bills-es.json'); +const bySlug = new Map(bills.map((b) => [slugOf(b), b])); +const billIndex = buildBillIndex(bills); +const cache = loadCache(); + +const todayUTC = new Date().toISOString().slice(0, 10); +if (!cache.dailyDecodes || cache.dailyDecodes.date !== todayUTC) { + cache.dailyDecodes = { date: todayUTC, count: 0 }; // roll over at UTC midnight +} + +console.log(`newsdesk: fetching ${SOURCES.length} feeds`); +const results = await Promise.allSettled(SOURCES.map(fetchFeed)); +const items = []; +results.forEach((r, i) => { + if (r.status === 'fulfilled') { + items.push(...r.value); + console.log(` ${SOURCES[i].name}: ${r.value.length} items`); + } else { + console.error(` ${SOURCES[i].name} FAILED: ${r.reason?.message ?? r.reason}`); + } +}); + +// Dedupe against the seen-headlines cache: skip anything already processed +// in a previous run (see the header comment's Dedupe section). +const newItems = items.filter((it) => !cache.seen.has(hashHeadline(it.title, it.outlet))); +console.log(`${items.length} headlines fetched, ${newItems.length} new (not previously seen)`); + +const citationSlugs = new Set(); +const t3Batch = []; +const t3Items = []; // parallel to t3Batch +const localOutletsBySlug = new Map(); // this run's t2/t3 outlet contributions, per slug + +for (const it of newItems) { + const citations = findCitations(it.title); + if (citations.length > 0) { + for (const c of citations) citationSlugs.add(c.slug); + continue; // citation tier wins outright - no need to also run t2/t3 + } + const local = matchLocal(it.title, billIndex); + if (local?.tier === 't2') { + if (!localOutletsBySlug.has(local.slug)) localOutletsBySlug.set(local.slug, new Set()); + localOutletsBySlug.get(local.slug).add(it.outlet ?? 'unknown'); + } else if (local?.tier === 'ambiguous' && looksLegislative(it.title) && t3Batch.length < T3_MAX_HEADLINES) { + t3Batch.push({ title: it.title, candidates: local.candidates }); + t3Items.push(it); + } + // else: no local signal at all, or not legislative-looking - dropped. + // t2/t3 can only ever resolve to a bill already in the corpus, by + // construction (see newsdesk-match.mjs's header comment), so a headline + // about a genuinely brand-new bill with no citation is unmatchable here. +} + +const t3Results = await resolveWithHaiku(anthropic, t3Batch); +console.log(`t3: ${t3Batch.length} headline(s) batched${t3Batch.length ? '' : ' (skipped - empty batch)'}, ${t3Results.size} resolved`); +for (const [i, slug] of t3Results) { + const it = t3Items[i]; + if (!localOutletsBySlug.has(slug)) localOutletsBySlug.set(slug, new Set()); + localOutletsBySlug.get(slug).add(it.outlet ?? 'unknown'); +} + +// Merge this run's t2/t3 outlet contributions into the persisted pending +// state, THEN decide fires - corroboration accumulates across runs rather +// than resetting hourly (decideFires's header comment has the reasoning). +for (const [slug, outlets] of localOutletsBySlug) { + const merged = new Set(cache.pendingOutlets[slug] ?? []); + for (const o of outlets) merged.add(o); + cache.pendingOutlets[slug] = [...merged]; +} +const pendingOutletsMap = new Map( + Object.entries(cache.pendingOutlets).map(([slug, outlets]) => [slug, new Set(outlets)]) +); +const { fired, reason } = decideFires(citationSlugs, pendingOutletsMap); +console.log(`fired this run: ${fired.size}${fired.size ? ' (' + [...fired].map((s) => `${s}:${reason.get(s)}`).join(', ') + ')' : ''}`); + +// ---- ON FIRE: refresh (free) or decode (gated by both caps) ---- +const forceSlugs = new Set(fired); // the press trigger's own corroboration stands in for the status gate +const outcomes = []; +let decodedThisRun = 0; +for (const slug of fired) { + const [type, number] = slug.split('-'); + const allowDecode = decodedThisRun < NEWSDESK_DECODE_CAP && cache.dailyDecodes.count < NEWSDESK_DAILY_DECODE_CAP; + const result = await syncOneBill({ type, number }, { allowDecode, forceSlugs, bills, es, bySlug, anthropic }); + outcomes.push(result.outcome); + if (result.outcome === 'added') { decodedThisRun++; cache.dailyDecodes.count++; } + if (result.outcome === 'refreshed' || result.outcome === 'added') { + delete cache.pendingOutlets[slug]; // corroboration spent - a future re-fire needs fresh corroboration + } + console.log(` ${slug}: ${result.outcome} (${reason.get(slug)})`); +} + +// ---- persist: cache always, data files only if something actually changed ---- +// Every headline this run touched (matched or not, fired or not) is marked +// seen so it isn't reprocessed next hour. The one accepted tradeoff: a +// citation-matched brand-new bill that hits BOTH decode caps this run +// ('budget' outcome) still gets its headline marked seen, so it won't +// retrigger from that exact article next hour - but a genuinely newsworthy +// bill almost always accumulates fresh headlines hour over hour, and even +// absent that, the nightly sync's own priority gate (scripts/decode-gate.mjs) +// will pick it up within a day once it has real recorded motion. +for (const it of newItems) cache.seen.add(hashHeadline(it.title, it.outlet)); +saveCache(cache); + +if (anyDataChanged(outcomes)) { + writeFileSync('data/bills.json', JSON.stringify(bills)); + writeFileSync('data/bills-es.json', JSON.stringify(es)); + const refreshedCount = outcomes.filter((o) => o === 'refreshed').length; + const addedCount = outcomes.filter((o) => o === 'added').length; + console.log(`DONE: ${refreshedCount} refreshed, ${addedCount} added+decoded; corpus ${bills.length}`); +} else { + console.log('DONE: no data changes this run - nothing written (the workflow commit step will no-op)'); +} diff --git a/scripts/sync-bills.mjs b/scripts/sync-bills.mjs index 8dd8c02..e594cfb 100644 --- a/scripts/sync-bills.mjs +++ b/scripts/sync-bills.mjs @@ -8,10 +8,38 @@ * * Policy: * - Existing bills: status/action/urgency/tags refresh freely (no AI cost). - * - NEW bills are decode-before-publish: they enter the corpus only once - * their EN+ES summary and headline exist, so the feed never shows - * undecoded entries. At most MAX_NEW_DECODES per run (cost ceiling); - * the rest wait for the next night. + * - NEW bills are decode-before-publish AND priority-gated: a new bill only + * spends a decode if it clears the priority gate (real legislative + * motion — see scripts/decode-gate.mjs) or is explicitly force-listed. + * Bills that clear the gate enter the corpus only once their EN+ES + * summary and headline exist, so the feed never shows undecoded entries. + * At most MAX_NEW_DECODES per run (cost ceiling); the rest wait for the + * next night. + * + * PRIORITY DECODE GATE (2026-07-16, owner directive: reduce spend, focus on + * a priority set of legislation — "the majority of the 2,147 bills is junk + * with high odds of never going anywhere"). A brand-new bill is decoded + * ONLY if `decode-gate.mjs`'s `passesGate(status)` says so (markup or + * later — NOT mere "referred to committee", which the gate treats as no + * real motion; see that module's header comment for the full status- + * distribution numbers and reasoning behind the line). This is enforced in + * ONE place, `bill-decode.mjs`'s `syncOneBill`, shared by BOTH the + * recent-first pass and the ascending backlog pass below, so the gate can't + * drift between them. Gate-skipped bills are NOT stored anywhere and count + * as fully handled: the ascending pass's cursor advances past them exactly + * as if they'd been decoded — this is what drains the multi-week decode + * backlog nearly for free, since ~80% of the corpus never had a real + * chance of clearing MAX_NEW_DECODES anyway. If a gated bill later gets + * real legislative motion, Congress.gov bumps its updateDate past + * wherever the cursor then sits, so the update feed resurfaces it on a + * later run and the gate re-evaluates against its new status — nothing + * about being gated out once is permanent. + * + * FORCE_DECODE_SLUGS (comma-separated slugs, e.g. "hr-1234-119,s-45-119") + * bypasses the gate for exactly those slugs — for a manual/workflow_dispatch + * catch-up run, or set in-process by scripts/newsdesk.mjs when a headline + * trigger decides a brand-new bill is newsworthy enough to decode outside + * the gate's own status-based test (see decode-gate.mjs's parseForceSlugs). * * Two-pass fetch (2026-07-16, audit §5 item 2). Congress.gov is queried * TWICE per run, in this order: @@ -19,13 +47,14 @@ * ~100 most-recently-touched bills in the whole 119th Congress, no * cursor floor. Already-known bills refresh for free; brand-new bills * decode within a RESERVED sub-budget (RECENT_DECODE_RESERVE, carved - * OUT of MAX_NEW_DECODES, not additional). This exists because the - * ascending backlog scan below structurally reaches the newest bills - * LAST - on a night with a deep backlog (or a busy legislative day) a - * floor vote that just happened would otherwise lose the race against - * both MAX_UPDATES and MAX_NEW_DECODES every single night, which is - * exactly how HR 7378 (and the whole "worth a call" feed) went stale - * for weeks even on clean, successful runs (see the audit). + * OUT of MAX_NEW_DECODES, not additional) AND must clear the priority + * gate above. This exists because the ascending backlog scan below + * structurally reaches the newest bills LAST - on a night with a deep + * backlog (or a busy legislative day) a floor vote that just happened + * would otherwise lose the race against both MAX_UPDATES and + * MAX_NEW_DECODES every single night, which is exactly how HR 7378 + * (and the whole "worth a call" feed) went stale for weeks even on + * clean, successful runs (see the audit). * 2. Ascending backlog: `fromDateTime: lastSync, sort=updateDate+asc` - * unchanged from before, drains the historical backlog oldest-first * with whatever decode budget the recent-first pass didn't use. A bill @@ -43,30 +72,29 @@ * exactly the failure this preserves the fix for. */ import Anthropic from '@anthropic-ai/sdk'; -import { readFileSync, writeFileSync } from 'node:fs'; +import { writeFileSync } from 'node:fs'; +import { loadJSON, syncOneBill } from './bill-decode.mjs'; import { BILL_TYPES, CONGRESS, cg, fetchRecentlyUpdated, - mapStatus, - refreshBillFields, slugOf, - tagBill, updateSlug, - urgencyScore, } from './congress-fetch.mjs'; -import { generateSearchInputs } from './search-inputs.mjs'; +import { parseForceSlugs } from './decode-gate.mjs'; const MAX_UPDATES = Number(process.env.MAX_UPDATES ?? 500); -// Raised 40 -> 120 (2026-07-16, audit §5 item 1): live nightly logs showed -// 373-418 bills/night needing decode against a 40-bill budget, pinning the -// ascending-pass cursor (state.lastSync) weeks behind and starving newer -// bills of decode slots night after night. 120 doesn't fully clear that -// inflow alone (~$8-14/night at $0.07-0.15/bill) - see the two-pass fetch -// design note above for the fix that stops recency from losing the race -// structurally, independent of how large the budget is. -const MAX_NEW_DECODES = Number(process.env.MAX_NEW_DECODES ?? 120); +// Lowered 120 -> 60 (2026-07-16, priority-decode-gate spec): with the gate +// above now doing the REAL limiting (only ~20.5% of bills - markup or +// later - are even eligible to spend a decode), MAX_NEW_DECODES reverts to +// a pure safety ceiling rather than the primary cost control it was when +// every new bill was decode-eligible. 60 comfortably covers a busy night's +// worth of genuinely-moving bills (441 gate-eligible bills total in the +// corpus today) without needing the 120 headroom that existed only to +// out-run an unfiltered ~373-418/night inflow of mostly just-introduced, +// zero-motion bills. +const MAX_NEW_DECODES = Number(process.env.MAX_NEW_DECODES ?? 60); // The recent-first pass's fetch window (audit §5 item 2 / §4 Alt A) - same // rough size as the twice-daily hot-bills.mjs refresh pass. const RECENT_FETCH_LIMIT = Number(process.env.RECENT_FETCH_LIMIT ?? 100); @@ -75,19 +103,20 @@ const RECENT_FETCH_LIMIT = Number(process.env.RECENT_FETCH_LIMIT ?? 100); // the last ~100 updates leaves the full MAX_NEW_DECODES for the ascending // backlog pass; a night with several leaves proportionally less. const RECENT_DECODE_RESERVE = Number(process.env.RECENT_DECODE_RESERVE ?? 20); +// See the header comment above and decode-gate.mjs. Empty by default. +const forceSlugs = parseForceSlugs(process.env.FORCE_DECODE_SLUGS); const anthropic = new Anthropic({ maxRetries: 8 }); -// Sonnet 5's tokenizer runs ~30% more tokens than 4.6 for the same text, so -// max_tokens caps on its calls are sized up accordingly; thinking is disabled -// explicitly because Sonnet 5 defaults it ON when the field is omitted, which -// would add unbounded thinking spend to batch calls. -const MODEL = 'claude-sonnet-5'; -const bills = JSON.parse(readFileSync('data/bills.json', 'utf8')); -const es = JSON.parse(readFileSync('data/bills-es.json', 'utf8')); -const state = JSON.parse(readFileSync('data/sync-state.json', 'utf8')); +const bills = loadJSON('data/bills.json'); +const es = loadJSON('data/bills-es.json'); +const state = loadJSON('data/sync-state.json'); const bySlug = new Map(bills.map((b) => [slugOf(b), b])); +if (forceSlugs.size) { + console.log(`FORCE_DECODE_SLUGS active (gate bypassed for): ${[...forceSlugs].join(', ')}`); +} + // Congress.gov's bill-list `updateDate` field is date-only (e.g. "2026-06-04"), // not a full timestamp. Persisting it as-is breaks the next run's fromDateTime // query, which Congress.gov 400s on - the 2026-06-25/07-01 outage. Always @@ -96,223 +125,48 @@ function toISODateTime(d) { return /T/.test(d) ? d : `${d}T00:00:00Z`; } -// ---- AI decode (new bills only) ---- -async function fetchBillText(type, number) { - const data = await cg(`/bill/${CONGRESS}/${type}/${number}/text`); - const versions = data.textVersions ?? []; - for (const v of [...versions].reverse()) { - const fmt = (v.formats ?? []).find((f) => f.type === 'Formatted Text'); - if (fmt?.url) { - const res = await fetch(fmt.url); - if (!res.ok) continue; - const html = await res.text(); - return html.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim().slice(0, 60_000); - } - } - return null; -} - -const DECODE_TAGS = [ - 'HEADLINE_EN', 'HEADLINE_ES', - 'TLDR', 'WHAT', 'WHO', 'WHY', 'COST', 'COST_CHIPS', - 'ES_TLDR', 'ES_WHAT', 'ES_WHO', 'ES_WHY', 'ES_COST', 'ES_COST_CHIPS', 'ES_SUMMARY', -]; - -function parseTagged(text) { - const out = {}; - for (let i = 0; i < DECODE_TAGS.length; i++) { - const tag = DECODE_TAGS[i]; - const start = text.indexOf(`[${tag}]`); - if (start === -1) throw new Error(`missing [${tag}]`); - const next = DECODE_TAGS.slice(i + 1) - .map((t) => text.indexOf(`[${t}]`)) - .filter((x) => x > start); - const end = next.length ? Math.min(...next) : text.length; - out[tag] = text.slice(start + tag.length + 2, end).trim(); - } - return out; -} - -const normCost = (s) => (s === 'NONE' || !s ? null : s); - -function normChips(s) { - if (s === 'NONE' || !s) return null; - const chips = s.split('|').map((c) => c.trim()).filter(Boolean); - if (chips.length < 1 || chips.length > 3 || chips.some((c) => c.length > 48)) return null; - return chips; -} - -async function decode(bill, text) { - const sum = await anthropic.messages.create({ - model: MODEL, max_tokens: 900, thinking: { type: 'disabled' }, - messages: [{ role: 'user', content: `Explain this congressional bill in plain language for an everyday US resident (8th-grade reading level). 2-3 short paragraphs: what it actually does, and who it affects. Strictly nonpartisan, no advocacy, no preamble, no markdown. - -Bill: ${bill.bill_type.toUpperCase()} ${bill.bill_number} — ${bill.title} - -Full text (may be truncated): -${text ?? bill.title}` }], - }); - const ai_summary = sum.content[0].text.trim(); - - const rest = await anthropic.messages.create({ - model: MODEL, max_tokens: 3250, thinking: { type: 'disabled' }, - messages: [{ role: 'user', content: `From this plain-language bill summary, produce headlines, scannable sections, and a Spanish translation. - -Bill: ${bill.bill_type.toUpperCase()} ${bill.bill_number} -Summary: -${ai_summary} - -STRICT RULES: -- Use ONLY facts present in the summary. Never invent numbers, costs, or claims. -- Headlines: 45-90 chars, sentence case, factual news-desk style, varied construction (NOT "Topic — Consequence", avoid colons), never start with "Congress". Prioritize the most decision-relevant specifics: what it does, who it affects, what it costs, or where it stands. -- TLDR: one sentence, max 160 chars, the single most decision-relevant fact. -- WHAT: 1-3 sentences. WHO: 1-2. WHY: 1-2 sentences of neutral consequence, never benefits-framing. -- COST: 1-2 sentences ONLY if the summary contains spending/funding/fines/who-pays content; otherwise output exactly NONE (and ES_COST, COST_CHIPS, ES_COST_CHIPS all NONE too). -- COST_CHIPS: when COST exists, compress it to 2-3 chips separated by " | ", each a standalone fact fragment max 45 chars, sentence case, no period. Same count and order in ES_COST_CHIPS. If a fact can't fit 45 chars, output NONE for both chip tags (prose is the fallback). -- Spanish: natural Latin American Spanish, 8th-grade level; citations/numbers exact; agency names in English with a short gloss when helpful. ES_SUMMARY is the full summary translation. -- Plain text, no markdown. - -Output exactly this tagged format, each tag on its own line followed by its content: -[HEADLINE_EN] -[HEADLINE_ES] -[TLDR] -[WHAT] -[WHO] -[WHY] -[COST] -[COST_CHIPS] -[ES_TLDR] -[ES_WHAT] -[ES_WHO] -[ES_WHY] -[ES_COST] -[ES_COST_CHIPS] -[ES_SUMMARY]` }], - }); - const p = parseTagged(rest.content[0].text.trim()); - if (!p.HEADLINE_EN || !p.TLDR || !p.WHAT || !p.WHO || !p.WHY || !p.ES_SUMMARY) { - throw new Error('bad decode shape'); - } - return { - ai_summary, - ai_headline: p.HEADLINE_EN.slice(0, 110), - ai_sections: { - tldr: p.TLDR, what: p.WHAT, who: p.WHO, why: p.WHY, - cost: normCost(p.COST), costChips: normChips(p.COST_CHIPS), - }, - es_headline: p.HEADLINE_ES.slice(0, 110), - es_summary: p.ES_SUMMARY, - es_sections: { - tldr: p.ES_TLDR, what: p.ES_WHAT, who: p.ES_WHO, why: p.ES_WHY, - cost: normCost(p.ES_COST), costChips: normChips(p.ES_COST_CHIPS), - }, - }; -} - // ---- main ---- const since = state.lastSync; const runStart = new Date().toISOString(); console.log(`sync since ${since}`); -// Shared new-bill decode-budget counter - both passes below decrement into -// this ONE pool (RECENT_DECODE_RESERVE is a ceiling on the recent-first -// pass's share of it, not a separate allowance; see the header comment). +// Shared new-bill decode-budget counter and gate counter - both passes +// below decrement/increment into these ONE pools (RECENT_DECODE_RESERVE is +// a ceiling on the recent-first pass's share of `added`, not a separate +// allowance; see the header comment). let added = 0; let refreshed = 0; // combined total across both passes (log-only, not gated) +let gated = 0; // combined total across both passes - no real legislative motion +let newFailed = 0; // new-bill decode failures specifically (subset of `failed` below) -/** - * Fetch one bill's current detail and either refresh it (already in the - * corpus - free) or decode it as new (only if `allowDecode`). The one place - * both passes below do "turn a Congress.gov update item into a corpus - * mutation", so the decode-before-publish invariant and the refresh fields - * can't drift between the recent-first pass and the ascending backlog pass. - * Returns one of: - * 'refreshed' - an existing bill's fields were updated in place - * 'added' - a brand-new bill was decoded and pushed into the corpus - * 'budget' - a brand-new bill was found but `allowDecode` was false - * 'failed' - the fetch or decode threw; `isNew` tells the caller - * whether this was a new-bill decode failure (must retry) - * or an existing bill's transient refresh failure - * (idempotent, self-heals on its next update). - */ -async function syncOneBill(u, allowDecode) { - const type = u.type.toLowerCase(); - const slug = updateSlug(u); - try { - const { bill: d } = await cg(`/bill/${CONGRESS}/${type}/${u.number}`); - const existing = bySlug.get(slug); - if (existing) { - refreshBillFields(existing, d); - return { outcome: 'refreshed', slug }; - } - if (!allowDecode) return { outcome: 'budget', slug }; - const status = mapStatus(d.latestAction?.text); - const lastActionDate = d.latestAction?.actionDate ?? null; - const bill = { - full_identifier: slug, - congress_number: CONGRESS, - bill_type: type, - bill_number: Number(u.number), - title: d.title, - short_title: null, - ai_summary: null, ai_headline: null, - sponsor_bioguide_id: d.sponsors?.[0]?.bioguideId ?? null, - introduced_date: d.introducedDate ?? null, - last_action_date: lastActionDate, - last_action_text: d.latestAction?.text ?? null, - status, - issue_tags: tagBill(d.policyArea?.name), - policy_area: d.policyArea?.name ?? null, - urgency_score: urgencyScore(status, lastActionDate), - congress_gov_url: `https://www.congress.gov/bill/${CONGRESS}th-congress/${type === 'hr' ? 'house-bill' : type === 's' ? 'senate-bill' : type === 'hjres' ? 'house-joint-resolution' : 'senate-joint-resolution'}/${u.number}`, - }; - const text = await fetchBillText(type, u.number); - const dec = await decode(bill, text); - bill.ai_summary = dec.ai_summary; - bill.ai_headline = dec.ai_headline; - bill.ai_sections = dec.ai_sections; - // Search handles for the coverage sync (press names + subject query). - // Non-fatal: the backfill script sweeps up any misses. - try { - const si = await generateSearchInputs(anthropic, bill); - bill.press_names = si.press_names; - bill.news_query = si.news_query; - } catch (e) { - console.error(` search-inputs failed for ${slug}: ${e.message}`); - } - es[slug] = { headline: dec.es_headline, summary: dec.es_summary, sections: dec.es_sections }; - bills.push(bill); - bySlug.set(slug, bill); - return { outcome: 'added', slug }; - } catch (e) { - console.error(`FAIL ${slug}: ${e.message}`); - return { outcome: 'failed', slug, isNew: !bySlug.has(slug) }; - } -} +const ctxBase = { bills, es, bySlug, anthropic, forceSlugs }; // ---- Pass 1: recent-first (audit §5 item 2) ---------------------------- // Guarantees this run always sees the most recently-touched bills in // Congress, no matter how deep the ascending backlog is. `handledSlugs` -// tracks everything this pass successfully resolved so pass 2 can dedupe -// without re-fetching or re-decoding - see updateSlug/refreshBillFields. +// tracks everything this pass fully resolved (refreshed, added, OR gated - +// a gate verdict is a resolution too) so pass 2 can dedupe without +// re-fetching or re-deciding - see updateSlug/refreshBillFields. const handledSlugs = new Set(); const recentDecodeCap = Math.min(RECENT_DECODE_RESERVE, MAX_NEW_DECODES); console.log(`recent-first pass: fetching up to ${RECENT_FETCH_LIMIT} most-recently-updated bills (decode reserve ${recentDecodeCap})`); const recentBills = await fetchRecentlyUpdated(RECENT_FETCH_LIMIT); -let recentRefreshed = 0, recentAdded = 0, recentDeferred = 0, recentFailed = 0; +let recentRefreshed = 0, recentAdded = 0, recentGated = 0, recentDeferred = 0, recentFailed = 0; for (const u of recentBills) { - const result = await syncOneBill(u, added < recentDecodeCap); + const result = await syncOneBill(u, { ...ctxBase, allowDecode: added < recentDecodeCap }); if (result.outcome === 'refreshed') { refreshed++; recentRefreshed++; handledSlugs.add(result.slug); } else if (result.outcome === 'added') { added++; recentAdded++; handledSlugs.add(result.slug); + } else if (result.outcome === 'gated') { + gated++; recentGated++; handledSlugs.add(result.slug); } else if (result.outcome === 'budget') { - recentDeferred++; // new bill, reserve exhausted - left for pass 2 (same run) or next run + recentDeferred++; // new bill, gate cleared but reserve exhausted - left for pass 2 (same run) or next run } else { recentFailed++; // logged only; deliberately NOT folded into the abort check below } } -console.log(`recent-first pass: ${recentRefreshed} refreshed, ${recentAdded} added+decoded, ${recentDeferred} deferred (reserve exhausted), ${recentFailed} failed`); +console.log(`recent-first pass: ${recentRefreshed} refreshed, ${recentAdded} added+decoded, ${recentGated} gated (no real motion), ${recentDeferred} deferred (reserve exhausted), ${recentFailed} failed`); // ---- Pass 2: ascending backlog scan from the cursor --------------------- // Unchanged shape from before the two-pass fetch - see the header comment. @@ -336,8 +190,13 @@ let queued = 0, failed = 0; // freeze it the instant we hit one that still needs work (decode budget // exhausted, or a new bill whose decode failed). A transient *refresh* failure // on a bill already in the corpus is idempotent and self-heals on its next -// update, so it doesn't freeze us - the old all-or-nothing freeze is what -// pinned lastSync for weeks and turned every run into a full window re-scan. +// update, so it doesn't freeze us. A GATED bill is likewise fully handled +// (not "still needs work") - it's deliberately not stored, and re-enters +// naturally via Congress.gov's own updateDate if it later moves - so it +// advances the cursor too. This dual property (transient-refresh-failure +// tolerance + gate-skip-is-handled) is what drains the backlog fast instead +// of freezing on the ~80% of bills that were never going to clear the gate +// anyway. let cursor = since; let frozen = false; for (const u of updated.slice(0, MAX_UPDATES)) { @@ -345,14 +204,16 @@ for (const u of updated.slice(0, MAX_UPDATES)) { let needsWork = false; if (handledSlugs.has(slug)) { // Already fully resolved by the recent-first pass this run - dedupe, - // don't re-fetch/re-decode. Resolved is resolved, so the cursor may + // don't re-fetch/re-decide. Resolved is resolved, so the cursor may // still advance over it exactly as if pass 2 had handled it itself. } else { - const result = await syncOneBill(u, added < MAX_NEW_DECODES); + const result = await syncOneBill(u, { ...ctxBase, allowDecode: added < MAX_NEW_DECODES }); if (result.outcome === 'refreshed') { refreshed++; } else if (result.outcome === 'added') { added++; + } else if (result.outcome === 'gated') { + gated++; // real legislative motion absent - fully handled, NOT queued/frozen } else if (result.outcome === 'budget') { queued++; // decode budget exhausted; revisit next run needsWork = true; @@ -360,7 +221,7 @@ for (const u of updated.slice(0, MAX_UPDATES)) { failed++; // A new bill that failed to decode must be retried; a failed refresh of // a known bill is idempotent and re-touches on its next update. - if (result.isNew) needsWork = true; + if (result.isNew) { needsWork = true; newFailed++; } } } if (needsWork) frozen = true; @@ -376,7 +237,15 @@ state.lastRun = runStart; writeFileSync('data/bills.json', JSON.stringify(bills)); writeFileSync('data/bills-es.json', JSON.stringify(es)); writeFileSync('data/sync-state.json', JSON.stringify(state, null, 2)); -console.log(`DONE: ${refreshed} refreshed, ${added} added+decoded, ${queued} queued for next run, ${failed} failed; corpus ${bills.length}`); + +// New bills seen this run, deduped across both passes: every new-bill slug +// this run touched resolves to exactly one of added/gated/queued/newFailed +// by the time we get here (a pass-1 'budget' deferral that pass 2 later +// resolves is NOT double-counted - see recentDeferred's comment above). +const newSeen = added + gated + queued + newFailed; +console.log( + `DONE: ${refreshed} refreshed, ${added} added+decoded, ${gated} gated (no real legislative motion), ${queued} queued for next run, ${failed} failed (${newFailed} new); new bills seen this run: ${newSeen}; corpus ${bills.length}` +); // Mostly-failed run: don't let CI commit garbage. Scoped to the ascending // pass's own failed/updated.length exactly as before the two-pass fetch - // the recent-first pass's (much smaller, logged-separately) failures don't diff --git a/tests/decode-gate.unit.spec.ts b/tests/decode-gate.unit.spec.ts new file mode 100644 index 0000000..1fd333e --- /dev/null +++ b/tests/decode-gate.unit.spec.ts @@ -0,0 +1,66 @@ +import { expect, test } from '@playwright/test'; +// Pure, I/O-free module (no CONGRESS_API_KEY/ANTHROPIC_API_KEY needed) - see +// scripts/decode-gate.mjs's header comment for the full status-distribution +// numbers and the reasoning behind the chosen gate line. +import { GATE_PASS_STATUSES, parseForceSlugs, passesGate } from '../scripts/decode-gate.mjs'; +import { mapStatus } from '../scripts/congress-fetch.mjs'; + +/* + * These tests PIN the priority decode gate's status table. If a status here + * surprises you, the gate line changed - retune deliberately and update the + * distribution-numbers comment in decode-gate.mjs alongside the pin. + */ + +test.describe('passesGate (status table)', () => { + test('committee does NOT pass - mere referral dominates this bucket (92.2% sampled)', () => { + expect(passesGate('committee')).toBe(false); + }); + + test('markup and later all pass - real legislative motion', () => { + expect(passesGate('markup')).toBe(true); + expect(passesGate('floor_vote')).toBe(true); + expect(passesGate('passed_chamber')).toBe(true); + expect(passesGate('conference')).toBe(true); + expect(passesGate('signed')).toBe(true); + expect(passesGate('vetoed')).toBe(true); + }); + + test('unknown/introduced status does not pass', () => { + expect(passesGate('introduced')).toBe(false); + expect(passesGate('some_future_status')).toBe(false); + expect(passesGate('')).toBe(false); + expect(passesGate(undefined)).toBe(false); + }); + + test('GATE_PASS_STATUSES is exactly the 6 real-motion statuses', () => { + expect([...GATE_PASS_STATUSES].sort()).toEqual( + ['conference', 'floor_vote', 'markup', 'passed_chamber', 'signed', 'vetoed'] + ); + }); + + test('hyphenated "Mark-up" action text maps to markup and clears the gate — 133 live bills depend on this spelling', () => { + expect(mapStatus('Committee Consideration and Mark-up Session Held')).toBe('markup'); + expect(mapStatus('Subcommittee Consideration and Markup Session Held')).toBe('markup'); + expect(passesGate(mapStatus('Committee Consideration and Mark-up Session Held'))).toBe(true); + // plain referral still reads as committee and stays gated + expect(mapStatus('Referred to the Committee on Energy and Commerce')).toBe('committee'); + }); +}); + +test.describe('parseForceSlugs', () => { + test('parses a comma-separated list, trims and lower-cases', () => { + expect(parseForceSlugs('HR-1234-119, s-45-119 ,hjres-9-119')).toEqual( + new Set(['hr-1234-119', 's-45-119', 'hjres-9-119']) + ); + }); + + test('empty/undefined/whitespace-only input yields an empty set', () => { + expect(parseForceSlugs(undefined)).toEqual(new Set()); + expect(parseForceSlugs('')).toEqual(new Set()); + expect(parseForceSlugs(' ')).toEqual(new Set()); + }); + + test('drops empty entries from stray commas', () => { + expect(parseForceSlugs('hr-1-119,,s-2-119,')).toEqual(new Set(['hr-1-119', 's-2-119'])); + }); +}); diff --git a/tests/newsdesk-match.unit.spec.ts b/tests/newsdesk-match.unit.spec.ts new file mode 100644 index 0000000..b5d7246 --- /dev/null +++ b/tests/newsdesk-match.unit.spec.ts @@ -0,0 +1,215 @@ +import { expect, test } from '@playwright/test'; +// Pure, I/O-free module (no CONGRESS_API_KEY/ANTHROPIC_API_KEY, no network) +// - see scripts/newsdesk-match.mjs's header comment for the full match +// design this pins. +import { + anyDataChanged, + buildBillIndex, + decideFires, + findCitations, + hashHeadline, + looksLegislative, + matchLocal, + parseFeed, +} from '../scripts/newsdesk-match.mjs'; + +test.describe('findCitations (t1 explicit bill-number citations)', () => { + test('H.R. 1234 resolves to hr-1234-119', () => { + expect(findCitations('House passes H.R. 1234 in bipartisan vote')).toEqual([ + { type: 'hr', number: '1234', slug: 'hr-1234-119' }, + ]); + }); + + test('HR1234 (no punctuation/space) resolves the same way', () => { + expect(findCitations('Senate committee advances HR1234')).toEqual([ + { type: 'hr', number: '1234', slug: 'hr-1234-119' }, + ]); + }); + + test('S. 567 resolves to s-567-119', () => { + expect(findCitations('Lawmakers debate S. 567 funding measure')).toEqual([ + { type: 's', number: '567', slug: 's-567-119' }, + ]); + }); + + test('H. Res. 12 is NOT tracked - must not match (simple House resolution, not hr/s/hjres/sjres)', () => { + expect(findCitations('A new H. Res. 12 honors the local team')).toEqual([]); + }); + + test('"US 567" must not match S. - no word boundary inside "US"', () => { + expect(findCitations('US 567 highway expansion project moves forward')).toEqual([]); + }); + + test('H.J.Res. 45 and the bare HJRES form both resolve to hjres-45-119', () => { + expect(findCitations('H.J.Res. 45 disapproval resolution passes House')).toEqual([ + { type: 'hjres', number: '45', slug: 'hjres-45-119' }, + ]); + expect(findCitations('HJRES 45 clears procedural hurdle')).toEqual([ + { type: 'hjres', number: '45', slug: 'hjres-45-119' }, + ]); + }); + + test('S.J.Res. 9 and the glued SJRES9 form both resolve to sjres-9-119', () => { + expect(findCitations('S.J.Res. 9 heads to the floor')).toEqual([ + { type: 'sjres', number: '9', slug: 'sjres-9-119' }, + ]); + expect(findCitations('SJRES9 gets a vote')).toEqual([ + { type: 'sjres', number: '9', slug: 'sjres-9-119' }, + ]); + }); + + test('multiple distinct citations in one headline are all found, deduped', () => { + expect(findCitations('House passes H.R. 1234 while Senate weighs S. 45')).toEqual([ + { type: 'hr', number: '1234', slug: 'hr-1234-119' }, + { type: 's', number: '45', slug: 's-45-119' }, + ]); + }); + + test('case-insensitive', () => { + expect(findCitations('hr1234 trending on social media')).toEqual([ + { type: 'hr', number: '1234', slug: 'hr-1234-119' }, + ]); + }); + + test('no citation-shaped text yields an empty array', () => { + expect(findCitations('Local bakery wins county fair blue ribbon')).toEqual([]); + }); +}); + +const BILLS = [ + { bill_type: 'hr', bill_number: 8463, congress_number: 119, title: 'Prevent Government Fraud Act of 2026', press_names: ['SAVE Act'] }, + { bill_type: 's', bill_number: 180, congress_number: 119, title: 'Secondary Exposure Act', press_names: null }, + { bill_type: 'hr', bill_number: 99, congress_number: 119, title: 'A generic bill about roads and bridges', press_names: null }, +]; + +test.describe('matchLocal (t2 free token-overlap match)', () => { + const index = buildBillIndex(BILLS); + + test('a headline naming the press_name + title words confidently matches one bill', () => { + expect(matchLocal('Congress passes the SAVE Act to prevent government fraud', index)).toEqual({ + tier: 't2', + slug: 'hr-8463-119', + }); + }); + + test('a headline with weak, tied overlap across two bills is ambiguous (t3-bound), not a guess', () => { + const result = matchLocal('Secondary exposure concerns raised about bridges funding', index); + expect(result?.tier).toBe('ambiguous'); + if (!result || !('candidates' in result)) throw new Error('unreachable'); + expect((result.candidates ?? []).map((c: { slug: string }) => c.slug).sort()).toEqual(['hr-99-119', 's-180-119']); + }); + + test('a headline with no meaningful overlap matches nothing', () => { + expect(matchLocal('Local weather turns cooler this weekend', index)).toBeNull(); + }); +}); + +test.describe('looksLegislative (t3 batch gate)', () => { + test('legislative-signal headlines pass', () => { + expect(looksLegislative('Senate passes major infrastructure bill')).toBe(true); + expect(looksLegislative('Committee advances markup on tax measure')).toBe(true); + }); + + test('non-legislative headlines do not, keeping the Haiku batch small', () => { + expect(looksLegislative('Local bakery wins award')).toBe(false); + expect(looksLegislative('')).toBe(false); + }); +}); + +test.describe('decideFires (the >=2-outlet corroboration rule)', () => { + test('a citation match fires off a SINGLE outlet - no corroboration required', () => { + const { fired, reason } = decideFires(new Set(['hr-1-119']), new Map()); + expect(fired).toEqual(new Set(['hr-1-119'])); + expect(reason.get('hr-1-119')).toBe('citation'); + }); + + test('a t2/t3 match from exactly ONE outlet does NOT fire', () => { + const outlets = new Map([['s-2-119', new Set(['cbsnews.com'])]]); + const { fired } = decideFires(new Set(), outlets); + expect(fired.size).toBe(0); + }); + + test('a t2/t3 match from TWO distinct outlets fires as corroborated', () => { + const outlets = new Map([['s-3-119', new Set(['cbsnews.com', 'foxnews.com'])]]); + const { fired, reason } = decideFires(new Set(), outlets); + expect(fired).toEqual(new Set(['s-3-119'])); + expect(reason.get('s-3-119')).toBe('corroborated'); + }); + + test('the SAME outlet appearing twice does not count as two outlets (Set dedupes)', () => { + const outlets = new Map([['s-4-119', new Set(['cbsnews.com'])]]); // caller already deduped by Set + const { fired } = decideFires(new Set(), outlets); + expect(fired.size).toBe(0); + }); + + test('citation and corroboration combine without double-counting or colliding', () => { + const citations = new Set(['hr-1-119']); + const outlets = new Map([ + ['hr-1-119', new Set(['cbsnews.com'])], // also has a lone t2 hit - citation reason wins + ['s-3-119', new Set(['thehill.com', 'npr.org'])], + ]); + const { fired, reason } = decideFires(citations, outlets); + expect(fired).toEqual(new Set(['hr-1-119', 's-3-119'])); + expect(reason.get('hr-1-119')).toBe('citation'); + expect(reason.get('s-3-119')).toBe('corroborated'); + }); +}); + +test.describe('anyDataChanged (the no-change-no-commit guard)', () => { + test('no fired bills at all -> no change', () => { + expect(anyDataChanged([])).toBe(false); + }); + + test('every outcome deferred/failed -> no change (nothing to commit)', () => { + expect(anyDataChanged(['budget', 'failed', 'budget'])).toBe(false); + }); + + test('a free refresh alone counts as a change', () => { + expect(anyDataChanged(['refreshed'])).toBe(true); + }); + + test('a decode alone counts as a change', () => { + expect(anyDataChanged(['budget', 'added', 'failed'])).toBe(true); + }); +}); + +test.describe('hashHeadline (seen-headlines dedupe key)', () => { + test('normalizes whitespace and case so near-identical entries collide on purpose', () => { + expect(hashHeadline('Some Title', 'cbsnews.com')).toBe(hashHeadline('some title ', 'CBSNEWS.com')); + }); + + test('different outlets for the same title hash differently (per-outlet dedupe)', () => { + expect(hashHeadline('Some Title', 'cbsnews.com')).not.toBe(hashHeadline('Some Title', 'foxnews.com')); + }); +}); + +test.describe('parseFeed (RSS/Atom, pure string parsing)', () => { + test('parses an RSS 2.0 ', () => { + const xml = 'Test Headlinehttps://example.com/aThu, 16 Jul 2026 12:00:00 GMT'; + expect(parseFeed(xml)).toEqual([ + { title: 'Test Headline', link: 'https://example.com/a', pubDate: 'Thu, 16 Jul 2026 12:00:00 GMT', source: null }, + ]); + }); + + test('parses an Atom with href-style ', () => { + const xml = 'Atom Title2026-07-16T12:00:00Z'; + expect(parseFeed(xml)).toEqual([ + { title: 'Atom Title', link: 'https://example.com/b', pubDate: '2026-07-16T12:00:00Z', source: null }, + ]); + }); + + test('extracts the per-article outlet domain from a Google-News-style tag', () => { + const xml = 'Bipartisan Medicare Bill Unites Congresshttps://news.google.com/rss/articles/XLegis1'; + expect(parseFeed(xml)[0].source).toBe('legis1.com'); + }); + + test('drops entries missing a title or link', () => { + const xml = 'No link herehttps://example.com/c'; + expect(parseFeed(xml)).toEqual([]); + }); + + test('decodes CDATA and HTML entities in titles', () => { + const xml = '<![CDATA[Cruz & Democrats push back]]>https://example.com/d'; + expect(parseFeed(xml)[0].title).toBe('Cruz & Democrats push back'); + }); +});