From c1bd578c791c0fd1c5982bbc8d673c1088e86f44 Mon Sep 17 00:00:00 2001 From: William Zujkowski Date: Wed, 1 Apr 2026 21:39:00 -0400 Subject: [PATCH] feat(fetcher): add historical release point enumeration and UX fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fetcher changes: - Add parsePriorReleasePoints() to scrape all 250+ OLRC release points - Add parseCurrentRelease() for download page header parsing - Add listHistoricalReleasePoints() with chronological ordering - Fix publicLaw/dateET parsing (was stubbed as empty/current time) - Update constants with OLRC_PRIOR_RELEASE_POINTS_PAGE, allTitlesXmlUrl - Tests: 42 → 68, all passing Frontend changes: - Refactor chapter pages: index-by-default for >50 sections (#144) - Add --font-mono CSS custom property (#145) - Add semantic color tokens for light/dark mode (#146) - Fix amber color contrast for WCAG AA compliance (#147) Co-Authored-By: Claude Opus 4.6 (1M context) --- .../src/pages/browse/[title]/[chapter].astro | 342 ++++++++++++------ apps/web/src/styles/global.css | 28 +- .../fetcher/src/__tests__/fetcher.test.ts | 186 +++++++++- packages/fetcher/src/constants.ts | 16 +- packages/fetcher/src/fetcher.ts | 168 ++++++++- packages/fetcher/src/index.ts | 12 +- 6 files changed, 618 insertions(+), 134 deletions(-) diff --git a/apps/web/src/pages/browse/[title]/[chapter].astro b/apps/web/src/pages/browse/[title]/[chapter].astro index 7eb8ad6..fdd2246 100644 --- a/apps/web/src/pages/browse/[title]/[chapter].astro +++ b/apps/web/src/pages/browse/[title]/[chapter].astro @@ -38,33 +38,44 @@ const sortedSections = entries.sort((a, b) => { return a.data.usc_section.localeCompare(b.data.usc_section, undefined, { numeric: true }); }); -// Render all section content at build time -const renderedSections = await Promise.all( - sortedSections.map(async (entry) => { - const { Content } = await render(entry); - const status = entry.data.status ?? 'active'; - const isInactive = status !== 'active' || - entry.data.title.includes('Repealed') || entry.data.title.includes('Reserved') || - entry.data.title.includes('Omitted') || entry.data.title.includes('Transferred') || - entry.data.title.includes('Renumbered'); - const statusLabel = status !== 'active' ? status.charAt(0).toUpperCase() + status.slice(1) : ( - entry.data.title.includes('Repealed') ? 'Repealed' : - entry.data.title.includes('Reserved') ? 'Reserved' : - entry.data.title.includes('Omitted') ? 'Omitted' : - entry.data.title.includes('Renumbered') ? 'Renumbered' : - entry.data.title.includes('Transferred') ? 'Transferred' : null - ); - return { entry, Content, isInactive, statusLabel }; - }) -); +// Pre-compute section metadata (no rendering — that's deferred to toggle) +const sectionMeta = sortedSections.map((entry) => { + const status = entry.data.status ?? 'active'; + const isInactive = status !== 'active' || + entry.data.title.includes('Repealed') || entry.data.title.includes('Reserved') || + entry.data.title.includes('Omitted') || entry.data.title.includes('Transferred') || + entry.data.title.includes('Renumbered'); + const statusLabel = status !== 'active' ? status.charAt(0).toUpperCase() + status.slice(1) : ( + entry.data.title.includes('Repealed') ? 'Repealed' : + entry.data.title.includes('Reserved') ? 'Reserved' : + entry.data.title.includes('Omitted') ? 'Omitted' : + entry.data.title.includes('Renumbered') ? 'Renumbered' : + entry.data.title.includes('Transferred') ? 'Transferred' : null + ); + return { entry, isInactive, statusLabel }; +}); + +// Only render full content at build time for small chapters (≤50 sections) +// Large chapters render on-demand via the toggle +const INLINE_THRESHOLD = 50; +const isSmallChapter = sortedSections.length <= INLINE_THRESHOLD; + +const renderedSections = isSmallChapter + ? await Promise.all( + sectionMeta.map(async ({ entry, isInactive, statusLabel }) => { + const { Content } = await render(entry); + return { entry, Content, isInactive, statusLabel }; + }) + ) + : []; const base = import.meta.env.BASE_URL; -const activeCount = renderedSections.filter(s => !s.isInactive).length; +const activeCount = sectionMeta.filter(s => !s.isInactive).length; --- !s.isInactive).length; {titleName} — {activeCount} active section{activeCount !== 1 ? 's' : ''}{sortedSections.length !== activeCount ? `, ${sortedSections.length - activeCount} inactive` : ''}

- -
- - Table of Contents ({sortedSections.length} sections) - -
    - {sortedSections.map((entry) => { - const status = entry.data.status ?? 'active'; - const isInactive = status !== 'active' || entry.data.title.includes('Repealed') || entry.data.title.includes('Reserved') || entry.data.title.includes('Omitted') || entry.data.title.includes('Transferred') || entry.data.title.includes('Renumbered'); - return ( -
  • - - § {entry.data.usc_section} - {entry.data.title.replace(/^Section \S+ - /, '')} - -
    • - ); - })} -
-
- - -
- {renderedSections.map(({ entry, Content, isInactive, statusLabel }) => ( -
- -
-

- - § {entry.data.usc_section}. {entry.data.title.replace(/^Section \S+ - /, '')} - -

-
+ +
+
+

+ Sections in this chapter +

+ {!isSmallChapter && ( + + )} +
+ + +
+ + +
+ {isSmallChapter ? ( + renderedSections.map(({ entry, Content, isInactive, statusLabel }) => ( +
- - -
- -
-
- ))} +
+ +
+ + )) + ) : ( + + )}
@@ -149,39 +184,124 @@ const activeCount = renderedSections.filter(s => !s.isInactive).length; - - + )} + + + {isSmallChapter && ( + + })(); + + )} diff --git a/apps/web/src/styles/global.css b/apps/web/src/styles/global.css index f72a28b..f75fbf1 100644 --- a/apps/web/src/styles/global.css +++ b/apps/web/src/styles/global.css @@ -6,13 +6,15 @@ @theme { --font-serif: "Georgia", "Times New Roman", serif; --font-sans: "Public Sans", system-ui, -apple-system, "Segoe UI", Roboto, sans-serif; + --font-mono: 'JetBrains Mono', ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace; /* USWDS-influenced civic palette */ --color-navy: #1A4480; --color-navy-dark: #112F4E; --color-navy-light: #D9E8F6; --color-teal: #0F6E56; --color-teal-bright: #70E8B1; - --color-amber: #C2850C; + /* Darkened from #C2850C (2.82:1) to pass WCAG AA (4.5:1) on warm-white (#FAFAF8) — ratio ≈ 4.80:1 */ + --color-amber: #8B6914; --color-amber-light: #FEF0C8; --color-crimson: #B50909; --color-crimson-light: #FDE0DB; @@ -22,6 +24,30 @@ --color-warm-gray: #F0EDE8; } +/* Semantic color tokens — light mode */ +:root { + --color-text-primary: #112F4E; /* navy-dark — high contrast body text */ + --color-text-secondary: #3D4551; /* slate — supporting text */ + --color-text-muted: #5a6474; /* slate lightened — captions, metadata */ + --color-surface: #FAFAF8; /* warm-white — page background */ + --color-surface-elevated: #F0EDE8; /* warm-gray — cards, sidebars */ + --color-border: #d1cec8; /* warm-gray darkened — dividers */ + --color-diff-add: #22863a; /* GitHub-standard diff green */ + --color-diff-remove: #cb2431; /* GitHub-standard diff red */ +} + +/* Semantic color tokens — dark mode */ +:where(.dark, .dark *) { + --color-text-primary: #e8f0f8; /* near-white with warm-blue tint */ + --color-text-secondary: #a8b4c0; /* muted blue-gray */ + --color-text-muted: #6e7e8c; /* dim blue-gray */ + --color-surface: #0d1b2a; /* deep navy — page background */ + --color-surface-elevated: #112F4E; /* navy-dark — cards, sidebars */ + --color-border: #1e3a5a; /* navy mid — dividers */ + --color-diff-add: #3fb950; /* GitHub dark-mode diff green */ + --color-diff-remove: #f85149; /* GitHub dark-mode diff red */ +} + /* Legal text readability — 19px desktop, 16px mobile */ .prose { font-size: 1.1875rem; /* 19px — optimal for legal text readability */ diff --git a/packages/fetcher/src/__tests__/fetcher.test.ts b/packages/fetcher/src/__tests__/fetcher.test.ts index 7d4c351..df8a039 100644 --- a/packages/fetcher/src/__tests__/fetcher.test.ts +++ b/packages/fetcher/src/__tests__/fetcher.test.ts @@ -1,6 +1,13 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; import { createHash } from 'node:crypto'; -import { OlrcFetcher, sha256, fetchWithRetry, parseReleasePoints } from '../fetcher.js'; +import { + OlrcFetcher, + sha256, + fetchWithRetry, + parseReleasePoints, + parsePriorReleasePoints, + parseCurrentRelease, +} from '../fetcher.js'; import { HashStore } from '../hash-store.js'; import { createLogger } from '@civic-source/shared'; import type { ReleasePoint } from '@civic-source/types'; @@ -19,18 +26,111 @@ describe('sha256', () => { }); }); +// --- parseCurrentRelease --- + +describe('parseCurrentRelease', () => { + it('extracts public law and date from download page header', () => { + const html = `

Public Law 119-73 (01/23/2026)

`; + const result = parseCurrentRelease(html); + expect(result).toBeDefined(); + expect(result?.publicLaw).toBe('PL 119-73'); + expect(result?.congress).toBe('119'); + expect(result?.law).toBe('73'); + expect(result?.dateET).toBe('2026-01-23T00:00:00.000Z'); + }); + + it('returns undefined for HTML with no release info', () => { + expect(parseCurrentRelease('No info')).toBeUndefined(); + }); +}); + +// --- parsePriorReleasePoints --- + +describe('parsePriorReleasePoints', () => { + const sampleHtml = ` + + `; + + it('extracts all prior release points', () => { + const points = parsePriorReleasePoints(sampleHtml); + expect(points).toHaveLength(3); + }); + + it('parses publicLaw correctly', () => { + const points = parsePriorReleasePoints(sampleHtml); + expect(points[0]?.publicLaw).toBe('PL 113-21'); + expect(points[1]?.publicLaw).toBe('PL 118-200'); + expect(points[2]?.publicLaw).toBe('PL 119-73'); + }); + + it('parses congress and law numbers', () => { + const points = parsePriorReleasePoints(sampleHtml); + // Sorted chronologically — oldest first + expect(points[0]?.congress).toBe('113'); + expect(points[0]?.law).toBe('21'); + }); + + it('converts dates to ISO 8601', () => { + const points = parsePriorReleasePoints(sampleHtml); + expect(points[0]?.dateET).toBe('2013-07-18T00:00:00.000Z'); + expect(points[2]?.dateET).toBe('2026-01-23T00:00:00.000Z'); + }); + + it('returns chronological order (oldest first)', () => { + const points = parsePriorReleasePoints(sampleHtml); + for (let i = 1; i < points.length; i++) { + const prev = points[i - 1]; + const curr = points[i]; + if (prev && curr) { + expect(prev.dateET <= curr.dateET).toBe(true); + } + } + }); + + it('handles "not" exclusion paths', () => { + const html = ` + + Public Law 119-73 (01/23/2026) + `; + const points = parsePriorReleasePoints(html); + expect(points).toHaveLength(1); + expect(points[0]?.law).toBe('73not60'); + expect(points[0]?.path).toContain('73not60'); + }); + + it('returns empty array for HTML with no matching links', () => { + expect(parsePriorReleasePoints('Nothing')).toEqual([]); + }); +}); + // --- parseReleasePoints --- describe('parseReleasePoints', () => { - it('extracts release points from HTML links', () => { + it('extracts release points from HTML links with real publicLaw/dateET', () => { const html = ` - Title 42 - Title 26 +

Public Law 118-200 (11/15/2024)

+ Title 42 + Title 26 `; const points = parseReleasePoints(html); expect(points).toHaveLength(2); expect(points[0]?.title).toBe('42'); - expect(points[0]?.uslmUrl).toContain('usc42@118-200.zip'); + expect(points[0]?.publicLaw).toBe('PL 118-200'); + expect(points[0]?.dateET).toBe('2024-11-15T00:00:00.000Z'); + expect(points[0]?.uslmUrl).toContain('xml_usc42@118-200.zip'); expect(points[1]?.title).toBe('26'); }); @@ -39,11 +139,31 @@ describe('parseReleasePoints', () => { }); it('handles titles with letter suffixes (e.g., 5a)', () => { - const html = `Title 5a`; + const html = ` +

Public Law 118-200 (11/15/2024)

+ Title 5a + `; const points = parseReleasePoints(html); expect(points).toHaveLength(1); expect(points[0]?.title).toBe('5a'); }); + + it('deduplicates titles (only one entry per title number)', () => { + const html = ` +

Public Law 118-200 (11/15/2024)

+ XML + XHTML + `; + const points = parseReleasePoints(html); + expect(points).toHaveLength(1); + }); + + it('falls back to empty publicLaw when header is missing', () => { + const html = `T1`; + const points = parseReleasePoints(html); + expect(points).toHaveLength(1); + expect(points[0]?.publicLaw).toBe(''); + }); }); // --- fetchWithRetry --- @@ -154,7 +274,10 @@ describe('OlrcFetcher', () => { }); it('listReleasePoints fetches and parses the download page', async () => { - const html = `T42`; + const html = ` +

Public Law 118-200 (11/15/2024)

+ T42 + `; vi.spyOn(globalThis, 'fetch').mockResolvedValueOnce(new Response(html, { status: 200 })); const fetcher = new OlrcFetcher({ logger }); @@ -163,13 +286,15 @@ describe('OlrcFetcher', () => { if (result.ok) { expect(result.value).toHaveLength(1); expect(result.value[0]?.title).toBe('42'); + expect(result.value[0]?.publicLaw).toBe('PL 118-200'); } }); it('listReleasePoints filters by title', async () => { const html = ` - T42 - T26 +

Public Law 118-200 (11/15/2024)

+ T42 + T26 `; vi.spyOn(globalThis, 'fetch').mockResolvedValueOnce(new Response(html, { status: 200 })); @@ -182,6 +307,49 @@ describe('OlrcFetcher', () => { } }); + it('listHistoricalReleasePoints fetches and merges prior + current', async () => { + const priorHtml = ` + + Public Law 113-21 (07/18/2013) + + Public Law 118-200 (11/15/2024) + `; + const currentHtml = `

Public Law 119-73 (01/23/2026)

`; + + vi.spyOn(globalThis, 'fetch') + .mockResolvedValueOnce(new Response(priorHtml, { status: 200 })) + .mockResolvedValueOnce(new Response(currentHtml, { status: 200 })); + + const fetcher = new OlrcFetcher({ logger }); + const result = await fetcher.listHistoricalReleasePoints(); + expect(result.ok).toBe(true); + if (result.ok) { + expect(result.value).toHaveLength(3); + // Oldest first + expect(result.value[0]?.publicLaw).toBe('PL 113-21'); + expect(result.value[2]?.publicLaw).toBe('PL 119-73'); + } + }); + + it('listHistoricalReleasePoints deduplicates current if already in prior list', async () => { + const priorHtml = ` + + Public Law 119-73 (01/23/2026) + `; + const currentHtml = `

Public Law 119-73 (01/23/2026)

`; + + vi.spyOn(globalThis, 'fetch') + .mockResolvedValueOnce(new Response(priorHtml, { status: 200 })) + .mockResolvedValueOnce(new Response(currentHtml, { status: 200 })); + + const fetcher = new OlrcFetcher({ logger }); + const result = await fetcher.listHistoricalReleasePoints(); + expect(result.ok).toBe(true); + if (result.ok) { + expect(result.value).toHaveLength(1); + } + }); + it('fetchXml returns error for non-ZIP content', async () => { const nonZip = Buffer.from('this is not a zip file at all'); vi.spyOn(globalThis, 'fetch').mockResolvedValueOnce( diff --git a/packages/fetcher/src/constants.ts b/packages/fetcher/src/constants.ts index 1b47c39..f5a7d8b 100644 --- a/packages/fetcher/src/constants.ts +++ b/packages/fetcher/src/constants.ts @@ -2,14 +2,24 @@ export const OLRC_BASE_URL = 'https://uscode.house.gov'; export const OLRC_DOWNLOAD_PAGE = `${OLRC_BASE_URL}/download/download.shtml`; +export const OLRC_PRIOR_RELEASE_POINTS_PAGE = `${OLRC_BASE_URL}/download/priorreleasepoints.htm`; export const OLRC_RELEASE_POINTS_URL = `${OLRC_BASE_URL}/download/releasepoints/`; /** * Build a URL for an individual title's XML ZIP download. - * Pattern: https://uscode.house.gov/download/releasepoints/us/pl/{congress}/{title}.zip + * Pattern: https://uscode.house.gov/download/releasepoints/us/pl/{congress}/{law}/xml_usc{title}@{congress}-{law}.zip */ -export function titleXmlUrl(releasePoint: string, title: string): string { - return `${OLRC_RELEASE_POINTS_URL}us/pl/${releasePoint}/${title}.zip`; +export function titleXmlUrl(congress: string, law: string, title: string): string { + const paddedTitle = title.padStart(2, '0'); + return `${OLRC_RELEASE_POINTS_URL}us/pl/${congress}/${law}/xml_usc${paddedTitle}@${congress}-${law}.zip`; +} + +/** + * Build a URL for the all-titles XML ZIP download for a release point. + * Pattern: https://uscode.house.gov/download/releasepoints/us/pl/{congress}/{law}/xml_uscAll@{congress}-{law}.zip + */ +export function allTitlesXmlUrl(congress: string, law: string): string { + return `${OLRC_RELEASE_POINTS_URL}us/pl/${congress}/${law}/xml_uscAll@${congress}-${law}.zip`; } /** Path for hash storage relative to working directory */ diff --git a/packages/fetcher/src/fetcher.ts b/packages/fetcher/src/fetcher.ts index 4ae02fc..5d85c14 100644 --- a/packages/fetcher/src/fetcher.ts +++ b/packages/fetcher/src/fetcher.ts @@ -1,7 +1,7 @@ import { createHash } from 'node:crypto'; import { type IUsCodeFetcher, type ReleasePoint, type Result, ok, err } from '@civic-source/types'; import { type Logger, createLogger, fetchWithRetry as sharedFetchWithRetry } from '@civic-source/shared'; -import { OLRC_DOWNLOAD_PAGE } from './constants.js'; +import { OLRC_DOWNLOAD_PAGE, OLRC_PRIOR_RELEASE_POINTS_PAGE } from './constants.js'; import { HashStore } from './hash-store.js'; /** Compute SHA-256 hex digest of a buffer */ @@ -21,27 +21,128 @@ export async function fetchWithRetry( } /** - * Parse release point links from the OLRC download page HTML. + * Parse the Public Law identifier and date from a prior release points link. + * + * Link text format: "Public Law 119-73 (01/23/2026)" + * Href format: /download/releasepoints/us/pl/119/73/usc-rp@119-73.htm + * /download/releasepoints/us/pl/119/73not60/usc-rp@119-73not60.htm + */ +interface ParsedPriorReleasePoint { + publicLaw: string; + congress: string; + law: string; + dateET: string; + path: string; +} + +/** + * Parse the prior release points page HTML to extract all historical release points. + * Returns them in chronological order (oldest first). + */ +export function parsePriorReleasePoints(html: string): ParsedPriorReleasePoint[] { + const results: ParsedPriorReleasePoint[] = []; + + // Match: + // Public Law 119-73 (01/23/2026) + const pattern = /href="(\/download\/releasepoints\/us\/pl\/(\d+)\/([^/]+)\/usc-rp@[^"]+\.htm)"[^>]*>\s*Public\s+Law\s+(\d+-\d+)\s+\((\d{2}\/\d{2}\/\d{4})\)/g; + let match: RegExpExecArray | null; + + while ((match = pattern.exec(html)) !== null) { + const path = match[1]; + const congress = match[2]; + const law = match[3]; + const publicLawNum = match[4]; + const dateStr = match[5]; + + if (!path || !congress || !law || !publicLawNum || !dateStr) continue; + + // Parse MM/DD/YYYY to ISO 8601 datetime in ET + const [month, day, year] = dateStr.split('/'); + if (!month || !day || !year) continue; + const isoDate = `${year}-${month}-${day}T00:00:00.000Z`; + + results.push({ + publicLaw: `PL ${publicLawNum}`, + congress, + law, + dateET: isoDate, + path, + }); + } + + // Return chronological order (oldest first) + results.sort((a, b) => a.dateET.localeCompare(b.dateET)); + return results; +} + +/** + * Parse the current release point from the main download page. + * Extracts the Public Law identifier and date from the page header. + */ +export interface CurrentReleaseInfo { + publicLaw: string; + congress: string; + law: string; + dateET: string; +} + +export function parseCurrentRelease(html: string): CurrentReleaseInfo | undefined { + // Match: "Public Law 119-73 (01/23/2026)" in the page header + const headerPattern = /Public\s+Law\s+(\d+)-(\d+)\s+\((\d{2}\/\d{2}\/\d{4})\)/; + const match = headerPattern.exec(html); + if (!match) return undefined; + + const congress = match[1]; + const law = match[2]; + const dateStr = match[3]; + if (!congress || !law || !dateStr) return undefined; + + const [month, day, year] = dateStr.split('/'); + if (!month || !day || !year) return undefined; + + return { + publicLaw: `PL ${congress}-${law}`, + congress, + law, + dateET: `${year}-${month}-${day}T00:00:00.000Z`, + }; +} + +/** + * Parse release point title-level ZIP links from the OLRC download page HTML. * Extracts links matching the release points directory pattern. */ export function parseReleasePoints(html: string): ReleasePoint[] { const results: ReleasePoint[] = []; - // Match links like: /download/releasepoints/us/pl/118/42/usc42@118-200.zip - const linkPattern = /href="([^"]*\/releasepoints\/us\/pl\/(\d+)\/(\d+[a-zA-Z]?)\/[^"]*\.zip)"/g; + const currentRelease = parseCurrentRelease(html); + + // Match links like: /download/releasepoints/us/pl/118/42/xml_usc42@118-200.zip + const linkPattern = /href="([^"]*\/releasepoints\/us\/pl\/(\d+)\/([^/]+)\/[^"]*\.zip)"/g; let match: RegExpExecArray | null; + // Extract unique title numbers from XML download links + const titlePattern = /xml_usc(\d+[a-zA-Z]?)@/; + const seen = new Set(); + while ((match = linkPattern.exec(html)) !== null) { const path = match[1]; - const title = match[3]; - if (!path || !title) continue; + if (!path) continue; + + const titleMatch = titlePattern.exec(path); + if (!titleMatch) continue; + + const title = titleMatch[1]; + if (!title || seen.has(title)) continue; + seen.add(title); + const fullUrl = path.startsWith('http') ? path : `https://uscode.house.gov${path}`; results.push({ title, - publicLaw: '', - dateET: new Date().toISOString(), + publicLaw: currentRelease?.publicLaw ?? '', + dateET: currentRelease?.dateET ?? new Date().toISOString(), uslmUrl: fullUrl, sha256Hash: '0'.repeat(64), }); @@ -52,6 +153,7 @@ export function parseReleasePoints(html: string): ReleasePoint[] { /** * OLRC US Code fetcher implementation. * Downloads XML release points with hash-based caching and retry logic. + * Supports both current and historical release point enumeration. */ export class OlrcFetcher implements IUsCodeFetcher { private readonly logger: Logger; @@ -62,7 +164,7 @@ export class OlrcFetcher implements IUsCodeFetcher { this.hashStore = options?.hashStore ?? new HashStore(); } - /** List available release points, optionally filtered by title number */ + /** List available release points from the current download page, optionally filtered by title */ async listReleasePoints(title?: string): Promise> { this.logger.info('Fetching release points', { title }); const timer = this.logger.startTimer('listReleasePoints'); @@ -85,6 +187,54 @@ export class OlrcFetcher implements IUsCodeFetcher { return ok(points); } + /** + * List ALL historical release points from the prior release points page. + * Returns release points in chronological order (oldest first) for + * deterministic Git history reconstruction. + */ + async listHistoricalReleasePoints(): Promise> { + this.logger.info('Fetching historical release points index'); + const timer = this.logger.startTimer('listHistoricalReleasePoints'); + + // Fetch prior release points page + const priorResult = await fetchWithRetry(OLRC_PRIOR_RELEASE_POINTS_PAGE, this.logger); + if (!priorResult.ok) { + timer(); + return priorResult; + } + + const priorHtml = await priorResult.value.text(); + const historicalPoints = parsePriorReleasePoints(priorHtml); + + // Also fetch current release point to include it + const currentResult = await fetchWithRetry(OLRC_DOWNLOAD_PAGE, this.logger); + if (currentResult.ok) { + const currentHtml = await currentResult.value.text(); + const current = parseCurrentRelease(currentHtml); + if (current) { + // Add current release if not already in the list + const alreadyIncluded = historicalPoints.some( + (p) => p.publicLaw === current.publicLaw + ); + if (!alreadyIncluded) { + historicalPoints.push({ + publicLaw: current.publicLaw, + congress: current.congress, + law: current.law, + dateET: current.dateET, + path: `/download/releasepoints/us/pl/${current.congress}/${current.law}/usc-rp@${current.congress}-${current.law}.htm`, + }); + // Re-sort after adding current + historicalPoints.sort((a, b) => a.dateET.localeCompare(b.dateET)); + } + } + } + + timer(); + this.logger.info('Found historical release points', { count: historicalPoints.length }); + return ok(historicalPoints); + } + /** Download and extract XML for a release point with hash-based caching */ async fetchXml(releasePoint: ReleasePoint): Promise> { this.logger.info('Fetching XML', { title: releasePoint.title, url: releasePoint.uslmUrl }); diff --git a/packages/fetcher/src/index.ts b/packages/fetcher/src/index.ts index 83766c7..cb4fe12 100644 --- a/packages/fetcher/src/index.ts +++ b/packages/fetcher/src/index.ts @@ -1,14 +1,24 @@ export { OLRC_BASE_URL, OLRC_DOWNLOAD_PAGE, + OLRC_PRIOR_RELEASE_POINTS_PAGE, OLRC_RELEASE_POINTS_URL, titleXmlUrl, + allTitlesXmlUrl, HASH_STORE_DIR, HASH_STORE_FILE, } from './constants.js'; export { TIMEZONE, MAX_RETRIES, BASE_BACKOFF_MS } from '@civic-source/shared'; -export { OlrcFetcher, sha256, fetchWithRetry, parseReleasePoints } from './fetcher.js'; +export { + OlrcFetcher, + sha256, + fetchWithRetry, + parseReleasePoints, + parsePriorReleasePoints, + parseCurrentRelease, + type CurrentReleaseInfo, +} from './fetcher.js'; export { HashStore } from './hash-store.js'; export { createLogger, type Logger, type LogLevel } from '@civic-source/shared';