diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 12471adc..10d8d7c7 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -4,7 +4,20 @@ on: pull_request: branches: [main] push: - branches: [main] + branches: ['**'] + schedule: + # Weekly Monday 14:00 UTC — governance review of internal + external links + - cron: "0 14 * * 1" + workflow_dispatch: + inputs: + check_external: + description: "Also check external (http/https) links" + type: boolean + default: true + verbose: + description: "Include normally ignored links in external checks" + type: boolean + default: false concurrency: group: ci-${{ github.ref }} @@ -13,6 +26,9 @@ concurrency: jobs: lint: name: Lint + if: >- + github.event_name == 'pull_request' || + (github.event_name == 'push' && github.ref == 'refs/heads/main') runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -96,3 +112,187 @@ jobs: - name: Accessibility audit working-directory: apps/site run: node a11y/full.js + + detect_link_apps: + name: Detect apps for link check + runs-on: ubuntu-latest + outputs: + apps: ${{ steps.apps.outputs.value }} + steps: + - uses: actions/checkout@v4 + + - name: Detect changed applications + if: github.event_name == 'push' && github.ref != 'refs/heads/main' + id: changes + uses: dorny/paths-filter@v3 + with: + # On branch pushes, inspect only files included in this push instead of + # every change accumulated on the branch since it diverged from main. + base: ${{ github.event.before }} + filters: | + site: + - 'apps/site/**' + - 'packages/ui-astro/**' + - 'packages/ui-react/**' + - 'packages/uswds-assets/**' + - 'packages/uswds-theme/**' + - 'package.json' + - 'package-lock.json' + - '.nvmrc' + - 'lychee.toml' + - 'scripts/check-links.mjs' + - 'scripts/link-exclusions.mjs' + consortium: + - 'apps/consortium/**' + - 'packages/google-api/**' + - 'packages/uswds-theme/**' + - 'package.json' + - 'package-lock.json' + - '.nvmrc' + - 'lychee.toml' + - 'scripts/check-links.mjs' + - 'scripts/link-exclusions.mjs' + docs: + - 'apps/docs/**' + - 'packages/uswds-theme/**' + - 'package.json' + - 'package-lock.json' + - '.nvmrc' + - 'lychee.toml' + - 'scripts/check-links.mjs' + - 'scripts/link-exclusions.mjs' + + - name: Select applications + id: apps + env: + EVENT_NAME: ${{ github.event_name }} + GIT_REF: ${{ github.ref }} + CHANGED_APPS: ${{ steps.changes.outputs.changes }} + run: | + if [[ "$EVENT_NAME" == "pull_request" || "$GIT_REF" == "refs/heads/main" || "$EVENT_NAME" == "schedule" || "$EVENT_NAME" == "workflow_dispatch" ]]; then + echo 'value=["site","consortium","docs"]' >> "$GITHUB_OUTPUT" + else + echo "value=${CHANGED_APPS:-[]}" >> "$GITHUB_OUTPUT" + fi + + link_check_internal: + name: Internal links (${{ matrix.app }}) + needs: detect_link_apps + if: needs.detect_link_apps.outputs.apps != '[]' + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + app: ${{ fromJSON(needs.detect_link_apps.outputs.apps) }} + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version-file: .nvmrc + cache: npm + + - run: npm ci + + - name: Build ${{ matrix.app }} + run: npm run build -w @bdc/${{ matrix.app }} + + - name: Check internal links + id: lychee + uses: lycheeverse/lychee-action@v2 + with: + fail: false + format: markdown + output: lychee-${{ matrix.app }}-internal.md + args: >- + --config lychee.toml + --offline + --root-dir ${{ github.workspace }}/apps/${{ matrix.app }}/dist + --no-progress + 'apps/${{ matrix.app }}/dist/**/*.html' + + - name: Upload report + if: always() + uses: actions/upload-artifact@v4 + with: + name: lychee-${{ matrix.app }}-internal + path: lychee-${{ matrix.app }}-internal.md + if-no-files-found: ignore + + link_check_external: + name: External links (${{ matrix.app }}) + if: >- + github.event_name == 'schedule' || + (github.event_name == 'workflow_dispatch' && inputs.check_external) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + app: [site, consortium] + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version-file: .nvmrc + cache: npm + + - run: npm ci + + - name: Build ${{ matrix.app }} + run: npm run build -w @bdc/${{ matrix.app }} + + - name: Generate default link exclusions + if: github.event_name != 'workflow_dispatch' || !inputs.verbose + run: >- + node scripts/link-exclusions.mjs + apps/${{ matrix.app }}/dist + .lycheeignore + + - name: Check external links + if: github.event_name != 'workflow_dispatch' || !inputs.verbose + id: lychee + uses: lycheeverse/lychee-action@v2 + with: + fail: false + format: markdown + output: lychee-${{ matrix.app }}-external.md + args: >- + --config lychee.toml + --scheme https + --scheme http + --root-dir ${{ github.workspace }}/apps/${{ matrix.app }}/dist + --cache + --max-cache-age 2d + --no-progress + 'apps/${{ matrix.app }}/dist/**/*.html' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Check all external links (verbose) + if: github.event_name == 'workflow_dispatch' && inputs.verbose + id: lychee_verbose + uses: lycheeverse/lychee-action@v2 + with: + fail: false + format: markdown + output: lychee-${{ matrix.app }}-external.md + args: >- + --config lychee.toml + --scheme https + --scheme http + --root-dir ${{ github.workspace }}/apps/${{ matrix.app }}/dist + --cache + --max-cache-age 2d + --no-progress + 'apps/${{ matrix.app }}/dist/**/*.html' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Upload report + if: always() + uses: actions/upload-artifact@v4 + with: + name: lychee-${{ matrix.app }}-external + path: lychee-${{ matrix.app }}-external.md + if-no-files-found: ignore diff --git a/.gitignore b/.gitignore index c7efa070..480fee72 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,5 @@ test-results # apps/freshdesk Pipfile Pipfile.lock + +.lycheecache \ No newline at end of file diff --git a/README.md b/README.md index 36f88601..f14e155d 100644 --- a/README.md +++ b/README.md @@ -176,9 +176,46 @@ Pull requests are automatically validated by CI, which runs: - **Build**: the app is built to catch compilation errors - **Tests**: Vitest runs automated test suites to validate application behavior - **Accessibility**: Playwright + axe-core audits every page against WCAG 2.0/2.1 AA (Section 508) +- **Link Check**: Lychee scans the built `site` and `consortium` HTML for broken internal links (weekly/manual runs also check external URLs) All checks must pass before a PR can be merged. +#### Local link checks + +The `links:*` scripts require the Lychee CLI to be installed and available on your `PATH`: + +```bash +# macOS (Homebrew) +brew install lychee + +# Linux (Ubuntu/Snap) +sudo snap install lychee + +# Verify the installation +lychee --version +``` + +You can then check every app from the repository root: + +```bash +npm run links:offline +``` + +To check an individual app, run the same script from its directory (for example, `apps/site`): + +```bash +cd apps/site +npm run links:offline +``` + +Hostnames listed in `DEFAULT_IGNORED_HOSTNAMES` in `scripts/link-exclusions.mjs` are skipped by default. The link checker also resolves Bitly URLs and skips only those that redirect to a listed hostname, so new shortened meeting links require no additional configuration and other Bitly destinations are still checked. Add `--verbose` to include every link in an online check: + +```bash +npm run links:online -- --verbose +``` + +Manual CI runs provide the same behavior through the **Include normally ignored links in external checks** option. Scheduled checks continue to apply the default hostname list. + ### Before opening a PR - Review `/docs/architecture.md` diff --git a/apps/consortium/package.json b/apps/consortium/package.json index 242fac5a..1ead0e0e 100644 --- a/apps/consortium/package.json +++ b/apps/consortium/package.json @@ -6,7 +6,10 @@ "scripts": { "dev": "astro dev", "build": "astro build", - "preview": "astro preview" + "preview": "astro preview", + "links:offline": "node ../../scripts/check-links.mjs offline consortium", + "links:online": "node ../../scripts/check-links.mjs online consortium", + "links:unique": "node ../../scripts/check-links.mjs unique consortium" }, "dependencies": { "@astrojs/mdx": "^5.0.4", diff --git a/apps/docs/package.json b/apps/docs/package.json index f680a373..a36f285e 100644 --- a/apps/docs/package.json +++ b/apps/docs/package.json @@ -8,7 +8,10 @@ "dev:sync": "npm run sync:content && astro dev", "sync:content": "GITBOOK_ALLOW_OVERWRITE=1 node ./scripts/sync-gitbook.mjs", "build": "npm run sync:content && astro build", - "preview": "astro preview" + "preview": "astro preview", + "links:offline": "node ../../scripts/check-links.mjs offline docs", + "links:online": "node ../../scripts/check-links.mjs online docs", + "links:unique": "node ../../scripts/check-links.mjs unique docs" }, "dependencies": { "@astrojs/starlight": "^0.38.4", diff --git a/apps/site/package.json b/apps/site/package.json index 8237399e..f197fb0c 100644 --- a/apps/site/package.json +++ b/apps/site/package.json @@ -15,6 +15,9 @@ "a11y:page": "node a11y/page.js", "a11y:smoke": "astro build && playwright test a11y/smoke.test.ts", "a11y:full": "astro build && node a11y/full.js", + "links:offline": "node ../../scripts/check-links.mjs offline site", + "links:online": "node ../../scripts/check-links.mjs online site", + "links:unique": "node ../../scripts/check-links.mjs unique site", "test": "vitest run", "test:watch": "vitest", "test:ui": "vitest --ui" diff --git a/lychee.toml b/lychee.toml new file mode 100644 index 00000000..07a8fd70 --- /dev/null +++ b/lychee.toml @@ -0,0 +1,42 @@ +# Lychee config for checking links in built Astro output (apps/*/dist). +# See: https://lychee.cli.rs/ + +############################# Runtime ############################# + +verbose = "warning" +no_progress = true +max_concurrency = 14 +max_redirects = 10 +max_retries = 2 +timeout = 20 +retry_wait_time = 2 + +# Resolve directory links like /about/ to /about/index.html +index_files = ["index.html"] + +############################# Requests ############################ + +# Accept rate-limit responses so CI does not flake on busy hosts +accept = ["200..=299", "429"] + +method = "get" + +# Skip mailto: (common in footers; not useful for link validation) +include_mail = false + +############################# Cache ############################### + +# Enabled via --cache on scheduled/external runs +max_cache_age = "2d" + +############################# Exclusions ########################## + +exclude = [ + # Placeholder/CMS tokens left in migrated RFC content + '^http://\[sitetree_link', + # Fragment-only / empty hash navigation helpers + '^#$', +] + +# Only scan HTML from the build output (inputs are passed on the CLI) +extensions = ["html"] diff --git a/package.json b/package.json index 91c75da9..29639305 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,10 @@ "format": "biome format --write .", "test": "vitest run", "test:watch": "vitest", - "test:ui": "vitest --ui" + "test:ui": "vitest --ui", + "links:offline": "node scripts/check-links.mjs offline", + "links:online": "node scripts/check-links.mjs online", + "links:unique": "node scripts/check-links.mjs unique" }, "devDependencies": { "@biomejs/biome": "2.3.14", diff --git a/scripts/check-links.mjs b/scripts/check-links.mjs new file mode 100644 index 00000000..919b5a0c --- /dev/null +++ b/scripts/check-links.mjs @@ -0,0 +1,138 @@ +import { spawnSync } from 'node:child_process'; +import { existsSync, readdirSync, readFileSync } from 'node:fs'; +import { join, resolve } from 'node:path'; +import { + escapeRegExp, + findIgnoredRedirects, + ignoredHostnamePattern, +} from './link-exclusions.mjs'; + +const root = resolve(import.meta.dirname, '..'); +const [mode = 'offline', ...options] = process.argv.slice(2); +const verbose = options.includes('--verbose'); +const positionalOptions = options.filter((option) => option !== '--verbose'); +const requestedApp = positionalOptions[0]; + +if ( + !['offline', 'online', 'unique'].includes(mode) || + positionalOptions.length > 1 +) { + console.error( + 'Usage: node scripts/check-links.mjs [app] [--verbose]', + ); + process.exit(1); +} + +const appsDirectory = join(root, 'apps'); +const discoveredApps = readdirSync(appsDirectory, { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .map((entry) => { + const directory = join(appsDirectory, entry.name); + const packagePath = join(directory, 'package.json'); + + if (!existsSync(packagePath)) return null; + + const packageJson = JSON.parse(readFileSync(packagePath, 'utf8')); + if (!packageJson.scripts?.build) return null; + + return { directory, name: entry.name, workspace: packageJson.name }; + }) + .filter(Boolean); + +const apps = requestedApp + ? discoveredApps.filter( + (app) => app.name === requestedApp || app.workspace === requestedApp, + ) + : discoveredApps; + +if (requestedApp && apps.length === 0) { + console.error(`No buildable app workspace found for "${requestedApp}".`); + process.exit(1); +} + +function run(command, args, options = {}) { + const result = spawnSync(command, args, { + cwd: root, + encoding: 'utf8', + stdio: 'inherit', + ...options, + }); + + if (result.error?.code === 'ENOENT') { + console.error( + `Could not find ${command}. Install it and ensure it is on PATH.`, + ); + process.exit(1); + } + + return result; +} + +let hasFailures = false; + +for (const app of apps) { + console.log(`\nBuilding ${app.workspace}...`); + const build = run('npm', ['run', 'build', '--workspace', app.workspace]); + if (build.status !== 0) { + hasFailures = true; + continue; + } + + console.log(`\nChecking links in ${app.name}...`); + const args = [ + '--config', + join(root, 'lychee.toml'), + '--root-dir', + join(app.directory, 'dist'), + ]; + + if (mode === 'online') { + args.push( + '--scheme', + 'https', + '--scheme', + 'http', + '--cache', + '--max-cache-age', + '2d', + ); + } else { + args.push('--offline'); + } + + if (!verbose) { + args.push('--exclude', ignoredHostnamePattern()); + + if (mode === 'online') { + const redirects = await findIgnoredRedirects(join(app.directory, 'dist')); + console.log( + `Ignoring ${redirects.length} Bitly redirect(s) to configured hostnames.`, + ); + for (const url of redirects) { + args.push('--exclude', `^${escapeRegExp(url)}$`); + } + } + } + if (mode === 'unique') args.push('--format', 'compact'); + args.push(`apps/${app.name}/dist/**/*.html`); + + const check = run( + 'lychee', + args, + mode === 'unique' ? { stdio: 'pipe' } : undefined, + ); + + if (mode === 'unique') { + const output = `${check.stdout ?? ''}\n${check.stderr ?? ''}`; + const distUrl = new URL(`${join(app.directory, 'dist')}/`, 'file:').href; + const links = [...output.matchAll(/file:\/\/\/[^ )\n]+/g)] + .map(([url]) => url.replace(distUrl, '/')) + .filter((url, index, all) => all.indexOf(url) === index) + .sort(); + console.log(links.join('\n')); + } else if (check.status !== 0) { + hasFailures = true; + } +} + +if (hasFailures) process.exit(1); diff --git a/scripts/link-exclusions.mjs b/scripts/link-exclusions.mjs new file mode 100644 index 00000000..f5266358 --- /dev/null +++ b/scripts/link-exclusions.mjs @@ -0,0 +1,83 @@ +import { readdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +const bitlyPattern = /https?:\/\/(?:www\.)?bit\.ly\/[A-Za-z0-9._~/?#=&%;+-]+/g; + +export const DEFAULT_IGNORED_HOSTNAMES = ['zoom.us', 'zoomgov.com', 'hhs.gov']; + +function htmlFiles(directory) { + return readdirSync(directory, { withFileTypes: true }).flatMap((entry) => { + const path = join(directory, entry.name); + if (entry.isDirectory()) return htmlFiles(path); + return entry.isFile() && entry.name.endsWith('.html') ? [path] : []; + }); +} + +export function escapeRegExp(value) { + return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +function isIgnoredUrl(value) { + const hostname = new URL(value).hostname.toLowerCase(); + return DEFAULT_IGNORED_HOSTNAMES.some( + (ignored) => hostname === ignored || hostname.endsWith(`.${ignored}`), + ); +} + +export function ignoredHostnamePattern() { + const hostnames = DEFAULT_IGNORED_HOSTNAMES.map(escapeRegExp).join('|'); + return `^https?://([^/]+\\.)?(${hostnames})(/|$)`; +} + +export async function findIgnoredRedirects(directory) { + const urls = new Set(); + + for (const file of htmlFiles(directory)) { + for (const url of readFileSync(file, 'utf8').match(bitlyPattern) ?? []) { + urls.add(url.replaceAll('&', '&')); + } + } + + const results = await Promise.all( + [...urls].map(async (url) => { + try { + const response = await fetch(url, { + method: 'HEAD', + redirect: 'follow', + signal: AbortSignal.timeout(15_000), + }); + return isIgnoredUrl(response.url) ? url : null; + } catch { + return null; + } + }), + ); + + return results.filter(Boolean).sort(); +} + +if ( + process.argv[1] && + import.meta.url === pathToFileURL(process.argv[1]).href +) { + const directory = resolve(process.argv[2] ?? ''); + const ignoreFile = process.argv[3]; + + if (!process.argv[2] || !ignoreFile) { + console.error( + 'Usage: node scripts/link-exclusions.mjs ', + ); + process.exit(1); + } + + const redirects = await findIgnoredRedirects(directory); + const patterns = [ + ignoredHostnamePattern(), + ...redirects.map((url) => `^${escapeRegExp(url)}$`), + ]; + writeFileSync(resolve(ignoreFile), patterns.join('\n')); + console.log( + `Excluded configured hostnames and ${redirects.length} matching Bitly redirect(s).`, + ); +}