From 9da21fc09ab7cf8760b82ec629544f616d0191e5 Mon Sep 17 00:00:00 2001 From: Adam Date: Wed, 1 Jul 2026 03:17:34 +0200 Subject: [PATCH 1/5] refactor: remove dead code + accessibility & polish pass Cleanup (grep-confirmed 0-usage, no behavior change): - Remove parseShadow, detectTypeScaleRatio, parseComputedStyles, getElementPath, ColorToken.rgb/hsl, OPEN_SIDE_PANEL enum, dead GET/SET_PREFERENCES service-worker handlers; dedupe isPixelLensElement. Accessibility & polish: - focus-visible + aria-current on nav, aria-label on all icon-only buttons - prefers-reduced-motion (CSS media + GSAP guard helper) - text-dim contrast raised to AA (4.81:1) - unified logo (PixelLensLogo), real keyboard shortcuts (manifest suggested_key + mac) - brand accent left unchanged (user decision) --- manifest.json | 10 +++- src/background/service-worker.ts | 18 ------- src/content/scanner/ColorExtractor.ts | 4 +- src/content/scanner/DesignSystemBuilder.ts | 38 ++----------- src/content/scanner/TypographyExtractor.ts | 39 -------------- src/content/ui/ContentApp.tsx | 10 +--- src/content/ui/FloatingToolbar.tsx | 8 +++ src/lib/__tests__/colors.test.ts | 12 ++--- src/lib/__tests__/design-tokens.test.ts | 2 - src/lib/__tests__/dom-utils.test.ts | 47 +--------------- src/lib/css-parser.ts | 62 ---------------------- src/lib/dom-utils.ts | 38 ------------- src/popup/Popup.tsx | 39 +++++++++++--- src/sidepanel/App.tsx | 20 ++++--- src/sidepanel/components/CSSBlock.tsx | 1 + src/sidepanel/components/ColorSwatch.tsx | 6 ++- src/sidepanel/components/ExportButton.tsx | 3 ++ src/sidepanel/components/PixelLensLogo.tsx | 40 ++++++++++++++ src/sidepanel/components/ShadowPreview.tsx | 1 + src/sidepanel/reducedMotion.ts | 16 ++++++ src/sidepanel/styles/panel.css | 2 +- src/sidepanel/views/CrawlView.tsx | 2 + src/sidepanel/views/DesignSystemView.tsx | 1 + src/sidepanel/views/ExportView.tsx | 1 + src/sidepanel/views/MarkdownView.tsx | 2 + src/sidepanel/views/ScanView.tsx | 7 +++ src/styles/globals.css | 18 ++++++- src/types/design-system.ts | 4 +- src/types/messages.ts | 7 --- 29 files changed, 169 insertions(+), 289 deletions(-) create mode 100644 src/sidepanel/components/PixelLensLogo.tsx create mode 100644 src/sidepanel/reducedMotion.ts diff --git a/manifest.json b/manifest.json index c9c1d35..9a140b7 100644 --- a/manifest.json +++ b/manifest.json @@ -39,10 +39,16 @@ } }, "commands": { - "_execute_action": {}, + "_execute_action": { + "suggested_key": { + "default": "Ctrl+Shift+P", + "mac": "Command+Shift+P" + } + }, "toggle-inspect": { "suggested_key": { - "default": "Ctrl+Shift+L" + "default": "Ctrl+Shift+L", + "mac": "Command+Shift+L" }, "description": "Toggle inspect mode" } diff --git a/src/background/service-worker.ts b/src/background/service-worker.ts index 16ac668..fb717a8 100644 --- a/src/background/service-worker.ts +++ b/src/background/service-worker.ts @@ -159,24 +159,6 @@ chrome.runtime.onMessage.addListener((message: IncomingMessage, sender, sendResp }) return true // async response } - - case MessageType.GET_PREFERENCES: { - chrome.storage.sync.get('pixellens_preferences', (result) => { - sendResponse(result['pixellens_preferences'] || { - colorFormat: 'hex', - gridSize: 8, - theme: 'dark', - }) - }) - return true // async response - } - - case MessageType.SET_PREFERENCES: { - const prefs = payload as MessagePayloadMap[MessageType.SET_PREFERENCES] - chrome.storage.sync.set({ pixellens_preferences: prefs.preferences }) - sendResponse({ success: true }) - break - } } }) diff --git a/src/content/scanner/ColorExtractor.ts b/src/content/scanner/ColorExtractor.ts index 90fac81..132cfc1 100644 --- a/src/content/scanner/ColorExtractor.ts +++ b/src/content/scanner/ColorExtractor.ts @@ -1,6 +1,6 @@ // PixelLens — Color Extractor (extract and cluster colors from the page) -import { toRgb, toHsl, isTransparent, clusterColors, classifyColors, tryToHex } from '@/lib/colors' +import { isTransparent, clusterColors, classifyColors, tryToHex } from '@/lib/colors' import type { ColorToken } from '@/types/design-system' const COLOR_PROPS = ['color', 'background-color', 'border-color', 'outline-color'] as const @@ -44,8 +44,6 @@ export class ColorExtractor { const tokens: ColorToken[] = clustered.map((c) => ({ name: '', hex: c.hex, - rgb: toRgb(c.hex), - hsl: toHsl(c.hex), frequency: c.frequency, category: 'accent' as const, })) diff --git a/src/content/scanner/DesignSystemBuilder.ts b/src/content/scanner/DesignSystemBuilder.ts index 2daba9e..f2a6bcc 100644 --- a/src/content/scanner/DesignSystemBuilder.ts +++ b/src/content/scanner/DesignSystemBuilder.ts @@ -6,7 +6,6 @@ import type { TypographyToken, SpacingToken, ShadowToken, - ShadowParsed, BorderRadiusToken, } from '@/types/design-system' @@ -46,44 +45,15 @@ export class DesignSystemBuilder { // Deduplicate by raw value if (shadowSet.has(shadow)) continue - const parsed = this.parseShadow(shadow) - if (parsed) { - shadowSet.set(shadow, { value: shadow, parsed }) - } + // Store the raw computed value only. The previous parseShadow() output + // (ShadowToken.parsed) was never read by any consumer — ShadowPreview + // renders shadow.value directly — so the parse was dead + buggy work. + shadowSet.set(shadow, { value: shadow }) } return Array.from(shadowSet.values()) } - private parseShadow(shadow: string): ShadowParsed | null { - // Basic shadow parsing: - // Computed values always resolve to rgb() format - const rgbMatch = shadow.match(/(rgba?\([^)]+\))\s+(-?[\d.]+px)\s+(-?[\d.]+px)\s+([\d.]+px)\s*([\d.]+px)?/) - if (rgbMatch) { - return { - color: rgbMatch[1], - x: rgbMatch[2], - y: rgbMatch[3], - blur: rgbMatch[4], - spread: rgbMatch[5] || '0px', - } - } - - // Alternate order: offsets first then color - const altMatch = shadow.match(/(-?[\d.]+px)\s+(-?[\d.]+px)\s+([\d.]+px)\s*([\d.]+px)?\s+(rgba?\([^)]+\))/) - if (altMatch) { - return { - x: altMatch[1], - y: altMatch[2], - blur: altMatch[3], - spread: altMatch[4] || '0px', - color: altMatch[5], - } - } - - return null - } - private extractBorderRadius(elements: Element[]): BorderRadiusToken[] { const freqMap = new Map() diff --git a/src/content/scanner/TypographyExtractor.ts b/src/content/scanner/TypographyExtractor.ts index 4ae0ca2..2bc6ad0 100644 --- a/src/content/scanner/TypographyExtractor.ts +++ b/src/content/scanner/TypographyExtractor.ts @@ -8,8 +8,6 @@ const TEXT_TAGS = new Set([ 'td', 'th', 'caption', 'blockquote', ]) -const KNOWN_RATIOS = [1.067, 1.125, 1.2, 1.25, 1.333, 1.414, 1.5, 1.618] as const - export class TypographyExtractor { extract(elements: Element[]): TypographyToken[] { // Family → Map<"size|weight" → variant with count> @@ -70,41 +68,4 @@ export class TypographyExtractor { return tokens } - - detectTypeScaleRatio(tokens: TypographyToken[]): number | null { - // Collect all unique font sizes across all families - const sizes = new Set() - for (const token of tokens) { - for (const v of token.variants) { - const px = parseFloat(v.fontSize) - if (px > 0) sizes.add(px) - } - } - - const sorted = Array.from(sizes).sort((a, b) => a - b) - if (sorted.length < 3) return null - - // Compute ratios between consecutive sizes - const ratios: number[] = [] - for (let i = 1; i < sorted.length; i++) { - ratios.push(sorted[i] / sorted[i - 1]) - } - - // Find the median ratio - ratios.sort((a, b) => a - b) - const median = ratios[Math.floor(ratios.length / 2)] - - // Match to nearest known ratio - let closest: number = KNOWN_RATIOS[0] - let minDiff = Math.abs(median - closest) - for (const r of KNOWN_RATIOS) { - const diff = Math.abs(median - r) - if (diff < minDiff) { - minDiff = diff - closest = r - } - } - - return minDiff < 0.1 ? closest : null - } } diff --git a/src/content/ui/ContentApp.tsx b/src/content/ui/ContentApp.tsx index 1b187c8..77f1dbc 100644 --- a/src/content/ui/ContentApp.tsx +++ b/src/content/ui/ContentApp.tsx @@ -6,6 +6,7 @@ import { FloatingToolbar } from './FloatingToolbar' import { InspectorTooltip } from './InspectorTooltip' import { sendMessage } from '@/lib/messaging' import { MessageType } from '@/types/messages' +import { isPixelLensElement } from '@/lib/dom-utils' type ContentMode = 'off' | 'inspect' | 'measure' | 'grid' @@ -98,15 +99,6 @@ function ContentApp() { ) } -function isPixelLensElement(el: Element): boolean { - let node: Node | null = el - while (node) { - if ((node as HTMLElement).id === 'pixellens-host') return true - node = node.parentNode - } - return false -} - export function mountContentApp(container: HTMLElement): void { const root = createRoot(container) root.render() diff --git a/src/content/ui/FloatingToolbar.tsx b/src/content/ui/FloatingToolbar.tsx index c5cee9a..4711a5c 100644 --- a/src/content/ui/FloatingToolbar.tsx +++ b/src/content/ui/FloatingToolbar.tsx @@ -8,6 +8,7 @@ import { Scan, } from '@phosphor-icons/react' import gsap from 'gsap' +import { prefersReducedMotion } from '@/sidepanel/reducedMotion' type ContentMode = 'off' | 'inspect' | 'measure' | 'grid' @@ -44,6 +45,12 @@ export function FloatingToolbar({ const el = toolbarRef.current if (!el) return + if (prefersReducedMotion()) { + // Appear in place, no slide-up. + gsap.set(el, { y: 0, opacity: 1 }) + return + } + gsap.fromTo( el, { y: 30, opacity: 0 }, @@ -159,6 +166,7 @@ export function FloatingToolbar({ {buttons.map((btn) => ( v1.0.0 diff --git a/src/sidepanel/App.tsx b/src/sidepanel/App.tsx index ba7ad0d..7572a92 100644 --- a/src/sidepanel/App.tsx +++ b/src/sidepanel/App.tsx @@ -21,6 +21,8 @@ import HistoryView from './views/HistoryView' import MarkdownView from './views/MarkdownView' import CrawlView from './views/CrawlView' import SettingsView from './views/SettingsView' +import PixelLensLogo from './components/PixelLensLogo' +import { prefersReducedMotion } from './reducedMotion' const MAIN_TABS: { mode: PanelMode; label: string; icon: typeof MagnifyingGlass }[] = [ { mode: 'inspect', label: 'Inspect', icon: MagnifyingGlass }, @@ -107,6 +109,9 @@ function App() { if (isFirstRender.current) { gsap.set(indicator, { left: el.offsetLeft, width: el.offsetWidth, opacity: 1 }) isFirstRender.current = false + } else if (prefersReducedMotion()) { + // Reduced motion: snap the indicator to the new tab instead of sliding. + gsap.set(indicator, { left: el.offsetLeft, width: el.offsetWidth, opacity: 1 }) } else { gsap.set(indicator, { opacity: 1 }) gsap.to(indicator, { @@ -191,9 +196,7 @@ function App() { if (!hasHydrated) { return (
-
- P -
+
) } @@ -224,9 +227,7 @@ function App() { {/* Header */}
-
- P -
+

PixelLens

@@ -247,7 +248,8 @@ function App() { key={tab.mode} ref={(el) => { tabsRef.current[i] = el }} onClick={() => setMode(tab.mode)} - className={`flex items-center gap-1.5 px-3 pb-2 pt-1 text-[12px] font-medium transition-colors duration-200 ${ + aria-current={isActive ? 'page' : undefined} + className={`flex items-center gap-1.5 px-3 pb-2 pt-1 text-[12px] font-medium rounded-md transition-colors duration-200 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-panel-accent focus-visible:ring-offset-2 focus-visible:ring-offset-panel-bg ${ isActive ? 'text-panel-text' : 'text-panel-text-dim hover:text-panel-text' }`} > @@ -274,7 +276,9 @@ function App() {
+ + {/* Mode tabs with GSAP sliding indicator */} @@ -292,6 +320,12 @@ function App() { v1.0.0 + + setPaletteOpen(false)} + /> ) } diff --git a/src/sidepanel/components/CommandPalette.tsx b/src/sidepanel/components/CommandPalette.tsx new file mode 100644 index 0000000..ef3bfed --- /dev/null +++ b/src/sidepanel/components/CommandPalette.tsx @@ -0,0 +1,366 @@ +import { Fragment, useEffect, useMemo, useRef, useState } from 'react' +import { + MagnifyingGlass, + Scan, + Palette, + Export, + ClockCounterClockwise, + MarkdownLogo, + GlobeHemisphereWest, + GearSix, + type Icon, +} from '@phosphor-icons/react' +import type { PanelMode } from '../store' +import { prefersReducedMotion } from '../reducedMotion' + +/** + * One entry in the command palette. + * + * The registry is deliberately decoupled from the store: a command only carries + * its presentation (`label`/`icon`/`group`/`keywords`) plus a `run` callback — + * it doesn't know *how* the action is wired. That keeps it extensible. + * + * -- Adding commands from another feature ----------------------------------- + * Export a factory that returns `Command[]` (mirror `createNavigationCommands` + * below) and spread its result into the `commands` array assembled in + * `App.tsx`: + * + * const commands = useMemo( + * () => [ + * ...createNavigationCommands(setMode), + * ...createContrastCommands(setMode, ...), // future feature + * ...createCrawlZipCommands(...), // future feature + * ], + * [setMode], + * ) + * + * `keywords` are extra fuzzy-search terms (synonyms/aliases) that widen matches + * without cluttering the visible `label`. `group` drives the section headers + * shown when the query is empty; add new groups to `GROUP_ORDER` to control + * their position. + */ +export interface Command { + id: string + label: string + icon: Icon + group: string + keywords?: string[] + run: () => void +} + +// Section order when no query is typed. Groups missing from this list fall to +// the end in first-seen (registry) order. +const GROUP_ORDER: string[] = ['Analyze', 'Convert', 'Workspace'] + +/** + * The base registry: every panel destination, mapped to `setMode`. Grouped so + * the flat 8-way navigation reads as a mental model (analyze / convert / + * workspace) instead of a wall of icons. + */ +export function createNavigationCommands(setMode: (mode: PanelMode) => void): Command[] { + const go = (mode: PanelMode) => () => setMode(mode) + return [ + { id: 'nav-inspect', label: 'Inspect element', icon: MagnifyingGlass, group: 'Analyze', keywords: ['pick', 'cursor', 'hover', 'element'], run: go('inspect') }, + { id: 'nav-scan', label: 'Scan design system', icon: Scan, group: 'Analyze', keywords: ['analyze', 'extract', 'tokens'], run: go('scan') }, + { id: 'nav-design-system', label: 'Design system', icon: Palette, group: 'Analyze', keywords: ['palette', 'colors', 'fonts', 'spacing', 'shadows', 'tokens'], run: go('design-system') }, + { id: 'nav-markdown', label: 'Convert to Markdown', icon: MarkdownLogo, group: 'Convert', keywords: ['md', 'article', 'reader', 'content'], run: go('markdown') }, + { id: 'nav-crawl', label: 'Crawl site', icon: GlobeHemisphereWest, group: 'Convert', keywords: ['spider', 'multi-page', 'website', 'sitemap'], run: go('crawl') }, + { id: 'nav-export', label: 'Export tokens', icon: Export, group: 'Workspace', keywords: ['css', 'tailwind', 'json', 'download', 'variables'], run: go('export') }, + { id: 'nav-history', label: 'History', icon: ClockCounterClockwise, group: 'Workspace', keywords: ['recent', 'past', 'scans'], run: go('history') }, + { id: 'nav-settings', label: 'Settings', icon: GearSix, group: 'Workspace', keywords: ['preferences', 'config', 'options', 'theme'], run: go('settings') }, + ] +} + +/** True on macOS/iOS so the shortcut hint shows the Command key instead of Ctrl. */ +export function isMacPlatform(): boolean { + if (typeof navigator === 'undefined') return false + const platform = navigator.platform || navigator.userAgent || '' + return /mac|iphone|ipad|ipod/i.test(platform) +} + +/** + * Lightweight subsequence fuzzy matcher (no dependency). Returns a score where + * higher is better, or -1 when `query` is not an ordered subsequence of `text`. + * Rewards consecutive runs, matches at word boundaries, and shorter targets. + */ +function fuzzyScore(query: string, text: string): number { + const t = text.toLowerCase() + let ti = 0 + let score = 0 + let run = 0 + let prev = -2 + for (let qi = 0; qi < query.length; qi++) { + const found = t.indexOf(query[qi], ti) + if (found === -1) return -1 + run = found === prev + 1 ? run + 1 : 0 + score += 1 + run * 3 + if (found === 0 || t[found - 1] === ' ' || t[found - 1] === '-') score += 8 + prev = found + ti = found + 1 + } + return score - t.length * 0.05 +} + +/** Best fuzzy score for a command across its label and keywords. */ +function scoreCommand(query: string, cmd: Command): number { + const q = query.trim().toLowerCase() + if (q === '') return 0 + let best = fuzzyScore(q, cmd.label) + if (cmd.keywords) { + for (const kw of cmd.keywords) { + const s = fuzzyScore(q, kw) + // Keyword hits count, but rank just below an equal-strength label hit. + if (s >= 0) best = Math.max(best, s - 2) + } + } + return best +} + +function Kbd({ children }: { children: React.ReactNode }) { + return ( + + {children} + + ) +} + +const LISTBOX_ID = 'cmdpalette-listbox' +const optionId = (i: number) => `cmdpalette-option-${i}` + +interface CommandPaletteProps { + open: boolean + commands: Command[] + onClose: () => void +} + +/** + * Command palette opened by the platform shortcut. Modal overlay (fixed within + * the 320px panel), fuzzy search over every destination, full keyboard control + * (up/down navigate, Enter run, Esc close), focus trap on the input, and the + * ARIA combobox + listbox pattern. Entry animation is skipped under + * `prefers-reduced-motion`. + */ +export function CommandPalette({ open, commands, onClose }: CommandPaletteProps) { + const [query, setQuery] = useState('') + const [activeIndex, setActiveIndex] = useState(0) + const [entered, setEntered] = useState(false) + const inputRef = useRef(null) + const listRef = useRef(null) + const reduce = prefersReducedMotion() + + // Filter + order. Empty query: everything, grouped by GROUP_ORDER (stable sort + // preserves registry order within a group). Typed query: flat, best-first. + const results = useMemo(() => { + const q = query.trim() + if (q === '') { + const rank = (g: string) => { + const i = GROUP_ORDER.indexOf(g) + return i === -1 ? GROUP_ORDER.length : i + } + return [...commands].sort((a, b) => rank(a.group) - rank(b.group)) + } + return commands + .map((cmd) => ({ cmd, score: scoreCommand(q, cmd) })) + .filter((r) => r.score >= 0) + .sort((a, b) => b.score - a.score) + .map((r) => r.cmd) + }, [commands, query]) + + // Reset query + selection each time the palette opens. + useEffect(() => { + if (!open) return + setQuery('') + setActiveIndex(0) + inputRef.current?.focus() + }, [open]) + + // Best match is always the default target as the query changes. + useEffect(() => { + setActiveIndex(0) + }, [query]) + + // Entry transition: paint hidden, then flip on the next frame. Skipped (shown + // immediately) when the user asked to reduce motion. + useEffect(() => { + if (!open) { + setEntered(false) + return + } + if (reduce) { + setEntered(true) + return + } + const raf = requestAnimationFrame(() => setEntered(true)) + return () => cancelAnimationFrame(raf) + }, [open, reduce]) + + // Keep the active option scrolled into view during keyboard navigation. + useEffect(() => { + if (!open) return + listRef.current + ?.querySelector(`#${optionId(activeIndex)}`) + ?.scrollIntoView({ block: 'nearest' }) + }, [activeIndex, open]) + + if (!open) return null + + const runCommand = (cmd: Command | undefined) => { + if (!cmd) return + cmd.run() + onClose() + } + + const onKeyDown = (e: React.KeyboardEvent) => { + switch (e.key) { + case 'Escape': + e.preventDefault() + onClose() + break + case 'ArrowDown': + e.preventDefault() + setActiveIndex((i) => (results.length === 0 ? 0 : (i + 1) % results.length)) + break + case 'ArrowUp': + e.preventDefault() + setActiveIndex((i) => (results.length === 0 ? 0 : (i - 1 + results.length) % results.length)) + break + case 'Home': + e.preventDefault() + setActiveIndex(0) + break + case 'End': + e.preventDefault() + setActiveIndex(Math.max(0, results.length - 1)) + break + case 'Enter': + e.preventDefault() + runCommand(results[activeIndex]) + break + case 'Tab': + // Only the input is focusable: keep focus trapped on it. + e.preventDefault() + inputRef.current?.focus() + break + } + } + + const transition = reduce ? 'none' : 'opacity 150ms ease, transform 150ms ease' + + return ( +
+ {/* Backdrop */} + + ) +} + +export default CommandPalette diff --git a/src/sidepanel/views/MarkdownView.tsx b/src/sidepanel/views/MarkdownView.tsx index 80cacba..9405360 100644 --- a/src/sidepanel/views/MarkdownView.tsx +++ b/src/sidepanel/views/MarkdownView.tsx @@ -14,11 +14,13 @@ import { Image as ImageIcon, Table, Code, + Sparkle, } from '@phosphor-icons/react' import { usePanelStore } from '../store' import { sendMessage } from '@/lib/messaging' import { MessageType } from '@/types/messages' import { copyToClipboard, downloadMarkdown } from '@/lib/export' +import { buildLlmBundle } from '@/lib/llm-bundle' import type { MarkdownFrontmatter, MarkdownResult } from '@/types/markdown' import MarkdownPreview from '../components/MarkdownPreview' @@ -59,7 +61,9 @@ function MarkdownView() { const setResult = usePanelStore((s) => s.setMarkdownResult) const setLoading = usePanelStore((s) => s.setMarkdownLoading) const setError = usePanelStore((s) => s.setMarkdownError) + const designSystem = usePanelStore((s) => s.designSystem) const [copied, setCopied] = useState(false) + const [copiedLlm, setCopiedLlm] = useState(false) const handleGenerate = useCallback(async () => { setLoading(true) @@ -180,6 +184,13 @@ function MarkdownView() { downloadMarkdown(deriveFilename(fm), result.fullDocument) } + const handleCopyLlm = async () => { + const bundle = buildLlmBundle({ markdown: result, designSystem }) + await copyToClipboard(bundle) + setCopiedLlm(true) + setTimeout(() => setCopiedLlm(false), 1500) + } + return (
{/* Meta header — frontmatter + stats */} @@ -234,26 +245,40 @@ function MarkdownView() {
{/* Actions */} -
+
+ {/* Hero action — the headline feature: one paste-ready document for an AI. */} - +
+ + +
) From 96b716b095af1710ae76f74affaeb29256cd47a5 Mon Sep 17 00:00:00 2001 From: Adam Date: Wed, 1 Jul 2026 03:42:49 +0200 Subject: [PATCH 3/5] feat: WCAG contrast checker + crawl ZIP export - Contrast checker: new view scoring the extracted palette's text/background pairs against WCAG AA/AAA (text labels, not color-only), reachable via nav and the Cmd+K palette. - Crawl export as ZIP (fflate 0.8.3): one .md per page + index.md, alongside the existing single-file mode. Toggle in the Crawl view. - Unit tests 193 -> 209. --- package-lock.json | 7 + package.json | 1 + src/lib/__tests__/contrast.test.ts | 127 +++++++++++++ src/lib/__tests__/crawl-zip.test.ts | 82 ++++++++ src/lib/contrast.ts | 121 ++++++++++++ src/lib/crawl-zip.ts | 127 +++++++++++++ src/lib/storage.ts | 1 + src/sidepanel/App.tsx | 22 ++- src/sidepanel/components/CommandPalette.tsx | 45 ++++- src/sidepanel/store.ts | 10 + src/sidepanel/views/ContrastCheckerView.tsx | 198 ++++++++++++++++++++ src/sidepanel/views/CrawlView.tsx | 86 ++++++--- 12 files changed, 802 insertions(+), 25 deletions(-) create mode 100644 src/lib/__tests__/contrast.test.ts create mode 100644 src/lib/__tests__/crawl-zip.test.ts create mode 100644 src/lib/contrast.ts create mode 100644 src/lib/crawl-zip.ts create mode 100644 src/sidepanel/views/ContrastCheckerView.tsx diff --git a/package-lock.json b/package-lock.json index 3dc230b..a54ea0a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,6 +11,7 @@ "@joplin/turndown-plugin-gfm": "^1.0.67", "@phosphor-icons/react": "^2.1.10", "chroma-js": "^3.2.0", + "fflate": "^0.8.3", "gsap": "^3.15.0", "react": "^19.2.7", "react-dom": "^19.2.7", @@ -3819,6 +3820,12 @@ ], "license": "BSD-3-Clause" }, + "node_modules/fflate": { + "version": "0.8.3", + "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.8.3.tgz", + "integrity": "sha512-tbZNuJrLwGUp3zshBtdy4W+ORxZuIh8a5ilyIEQDC5rY1f3U20JMry0Ll3WBzU58EZKsEuJFXhb5gwv8CsPvgA==", + "license": "MIT" + }, "node_modules/file-entry-cache": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz", diff --git a/package.json b/package.json index 99a69d0..4d24c1e 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "@joplin/turndown-plugin-gfm": "^1.0.67", "@phosphor-icons/react": "^2.1.10", "chroma-js": "^3.2.0", + "fflate": "^0.8.3", "gsap": "^3.15.0", "react": "^19.2.7", "react-dom": "^19.2.7", diff --git a/src/lib/__tests__/contrast.test.ts b/src/lib/__tests__/contrast.test.ts new file mode 100644 index 0000000..ec6ea37 --- /dev/null +++ b/src/lib/__tests__/contrast.test.ts @@ -0,0 +1,127 @@ +import { describe, it, expect } from 'vitest' +import { + evaluateContrast, + buildContrastReport, + formatRatio, + WCAG_THRESHOLDS, +} from '../contrast' +import type { ColorToken, ColorCategory } from '@/types/design-system' + +const token = (hex: string, category: ColorCategory, frequency = 1): ColorToken => ({ + name: hex, + hex, + frequency, + category, +}) + +describe('evaluateContrast', () => { + it('scores black on white as the maximum 21:1 and passes every level', () => { + const v = evaluateContrast('#000000', '#ffffff') + expect(v.ratio).toBeCloseTo(21, 5) + expect(v.aaNormal).toBe(true) + expect(v.aaLarge).toBe(true) + expect(v.aaaNormal).toBe(true) + expect(v.aaaLarge).toBe(true) + expect(v.normalLevel).toBe('AAA') + expect(v.largeLevel).toBe('AAA') + }) + + it('scores identical colors as 1:1 and fails every level', () => { + const v = evaluateContrast('#123456', '#123456') + expect(v.ratio).toBeCloseTo(1, 5) + expect(v.aaNormal).toBe(false) + expect(v.aaLarge).toBe(false) + expect(v.normalLevel).toBe('fail') + expect(v.largeLevel).toBe('fail') + }) + + it('passes AA (not AAA) for normal text at ~4.54:1 (#767676 on white)', () => { + const v = evaluateContrast('#767676', '#ffffff') + expect(v.ratio).toBeGreaterThanOrEqual(WCAG_THRESHOLDS.aaNormal) + expect(v.ratio).toBeLessThan(WCAG_THRESHOLDS.aaaNormal) + expect(v.aaNormal).toBe(true) + expect(v.aaaNormal).toBe(false) + expect(v.normalLevel).toBe('AA') + // ratio >= 4.5 also clears AAA for large text. + expect(v.largeLevel).toBe('AAA') + }) + + it('passes large text only at ~3.23:1 (#8f8f8f on white)', () => { + const v = evaluateContrast('#8f8f8f', '#ffffff') + expect(v.aaNormal).toBe(false) + expect(v.aaLarge).toBe(true) + expect(v.aaaLarge).toBe(false) + expect(v.normalLevel).toBe('fail') + expect(v.largeLevel).toBe('AA') + }) + + it('fails both sizes below 3:1 (#a0a0a0 on white)', () => { + const v = evaluateContrast('#a0a0a0', '#ffffff') + expect(v.aaLarge).toBe(false) + expect(v.normalLevel).toBe('fail') + expect(v.largeLevel).toBe('fail') + }) + + it('is symmetric in its two arguments', () => { + expect(evaluateContrast('#6366F1', '#ffffff').ratio).toBeCloseTo( + evaluateContrast('#ffffff', '#6366F1').ratio, + 6, + ) + }) +}) + +describe('formatRatio', () => { + it('renders two decimals', () => { + expect(formatRatio(4.5)).toBe('4.50') + expect(formatRatio(21)).toBe('21.00') + }) +}) + +describe('buildContrastReport', () => { + it('pairs each foreground against explicit background tokens, best contrast first', () => { + const colors = [ + token('#ffffff', 'background', 10), + token('#000000', 'text', 8), + token('#767676', 'neutral', 5), + token('#6366F1', 'primary', 6), + ] + const report = buildContrastReport(colors) + + expect(report).toHaveLength(1) + expect(report[0].background.hex).toBe('#ffffff') + // Background token is not evaluated as its own foreground. + expect(report[0].foregrounds.map((f) => f.token.hex)).toEqual([ + '#000000', + '#767676', + '#6366F1', + ]) + expect(report[0].foregrounds[0].verdict.aaaNormal).toBe(true) + }) + + it('skips a foreground identical to its background, dropping empty pairings', () => { + const colors = [token('#ffffff', 'background'), token('#ffffff', 'text')] + expect(buildContrastReport(colors)).toEqual([]) + }) + + it('falls back to luminance extremes when no background token is classified', () => { + const colors = [ + token('#000000', 'text'), + token('#767676', 'neutral'), + token('#6366F1', 'primary'), + ] + const report = buildContrastReport(colors) + expect(report.length).toBeGreaterThanOrEqual(1) + const inputHexes = new Set(colors.map((c) => c.hex)) + for (const pairing of report) { + expect(inputHexes.has(pairing.background.hex)).toBe(true) + // No pairing ever includes a foreground equal to its own background. + for (const fg of pairing.foregrounds) { + expect(fg.token.hex).not.toBe(pairing.background.hex) + } + } + }) + + it('returns nothing for an empty palette', () => { + expect(buildContrastReport([])).toEqual([]) + }) +}) diff --git a/src/lib/__tests__/crawl-zip.test.ts b/src/lib/__tests__/crawl-zip.test.ts new file mode 100644 index 0000000..77f003c --- /dev/null +++ b/src/lib/__tests__/crawl-zip.test.ts @@ -0,0 +1,82 @@ +import { describe, it, expect } from 'vitest' +import { unzipSync, strFromU8 } from 'fflate' +import { buildZipEntries, buildCrawlZip, crawlZipFilename } from '../crawl-zip' +import type { CrawlResult } from '@/types/crawl' + +function makeResult(): CrawlResult { + return { + origin: 'https://ex.com', + host: 'ex.com', + startUrl: 'https://ex.com', + pages: [ + { url: 'https://ex.com/a', title: 'Alpha', markdown: '# Alpha\n\nbody a', wordCount: 2 }, + // Same title as the first page -> slug must be de-duplicated. + { url: 'https://ex.com/b', title: 'Alpha', markdown: 'body b', wordCount: 1 }, + // Empty title -> positional fallback slug. + { url: 'https://ex.com/c', title: '', markdown: 'body c', wordCount: 1 }, + ], + skipped: ['https://ex.com/private'], + document: 'IGNORED single-file document', + generatedAt: '2026-07-01T00:00:00.000Z', + stats: { pageCount: 3, skippedCount: 1, bytes: 100, discovery: 'crawl' }, + } +} + +describe('buildZipEntries', () => { + it('emits index.md plus one de-duplicated file per page', () => { + const files = buildZipEntries(makeResult()) + const paths = Object.keys(files).sort() + expect(paths).toEqual([ + 'index.md', + 'pages/alpha-1.md', + 'pages/alpha.md', + 'pages/page-3.md', + ]) + }) + + it('links every page from index.md and carries global frontmatter', () => { + const files = buildZipEntries(makeResult()) + const index = files['index.md'] + expect(index).toContain('site: "ex.com"') + expect(index).toContain('pages: 3') + expect(index).toContain('skipped: 1') + expect(index).toContain('discovery: "crawl"') + // Table of contents references the exact file paths + source URLs. + expect(index).toContain('[Alpha](pages/alpha.md) — https://ex.com/a') + expect(index).toContain('[Alpha](pages/alpha-1.md) — https://ex.com/b') + expect(index).toContain('[Untitled](pages/page-3.md) — https://ex.com/c') + }) + + it('writes each page body with per-file frontmatter', () => { + const files = buildZipEntries(makeResult()) + const alpha = files['pages/alpha.md'] + expect(alpha).toContain('title: "Alpha"') + expect(alpha).toContain('url: "https://ex.com/a"') + expect(alpha).toContain('body a') + expect(files['pages/page-3.md']).toContain('body c') + }) +}) + +describe('buildCrawlZip', () => { + it('produces a real zip that round-trips through unzip', () => { + const result = makeResult() + const bytes = buildCrawlZip(result) + expect(bytes).toBeInstanceOf(Uint8Array) + expect(bytes.length).toBeGreaterThan(0) + + const unzipped = unzipSync(bytes) + const decoded: Record = {} + for (const [path, data] of Object.entries(unzipped)) decoded[path] = strFromU8(data) + + const expected = buildZipEntries(result) + expect(Object.keys(decoded).sort()).toEqual(Object.keys(expected).sort()) + expect(decoded['index.md']).toBe(expected['index.md']) + expect(decoded['pages/alpha-1.md']).toBe(expected['pages/alpha-1.md']) + }) +}) + +describe('crawlZipFilename', () => { + it('derives a friendly archive name from the host', () => { + expect(crawlZipFilename(makeResult())).toBe('ex.com-site.zip') + }) +}) diff --git a/src/lib/contrast.ts b/src/lib/contrast.ts new file mode 100644 index 0000000..3f23639 --- /dev/null +++ b/src/lib/contrast.ts @@ -0,0 +1,121 @@ +// PixelLens — WCAG Contrast Evaluation +// +// Pure, browser-free layer on top of the chroma-js wrappers in `colors.ts`. +// Turns the extracted color tokens of the latest scan into a set of relevant +// text/background pairings, each scored against the WCAG 2.x contrast ratio +// thresholds. Kept free of the DOM so it is fully testable. + +import chroma from 'chroma-js' +import { getContrastRatio } from './colors' +import type { ColorToken, ColorCategory } from '@/types/design-system' + +// WCAG 2.x minimum contrast ratios. +// AA normal text >= 4.5 · AA large text (>=18.66px bold / >=24px) >= 3 +// AAA normal text >= 7 · AAA large text >= 4.5 +export const WCAG_THRESHOLDS = { + aaNormal: 4.5, + aaLarge: 3, + aaaNormal: 7, + aaaLarge: 4.5, +} as const + +/** Best passing level for a given text size, or `'fail'` when nothing passes. */ +export type ContrastLevel = 'AAA' | 'AA' | 'fail' + +export interface ContrastVerdict { + /** Raw WCAG contrast ratio (1 -> 21). Not rounded — round at the display edge. */ + ratio: number + aaNormal: boolean + aaLarge: boolean + aaaNormal: boolean + aaaLarge: boolean + /** Best passing level for normal-size text. */ + normalLevel: ContrastLevel + /** Best passing level for large text. */ + largeLevel: ContrastLevel +} + +/** Score a foreground color against a background color per WCAG 2.x. */ +export function evaluateContrast(foreground: string, background: string): ContrastVerdict { + const ratio = getContrastRatio(foreground, background) + const aaNormal = ratio >= WCAG_THRESHOLDS.aaNormal + const aaLarge = ratio >= WCAG_THRESHOLDS.aaLarge + const aaaNormal = ratio >= WCAG_THRESHOLDS.aaaNormal + const aaaLarge = ratio >= WCAG_THRESHOLDS.aaaLarge + return { + ratio, + aaNormal, + aaLarge, + aaaNormal, + aaaLarge, + normalLevel: aaaNormal ? 'AAA' : aaNormal ? 'AA' : 'fail', + largeLevel: aaaLarge ? 'AAA' : aaLarge ? 'AA' : 'fail', + } +} + +/** WCAG contrast ratio formatted for display, e.g. `5.23`. */ +export function formatRatio(ratio: number): string { + return ratio.toFixed(2) +} + +/** WCAG relative luminance (0 -> 1); returns 0 for an unparseable color. */ +function luminance(hex: string): number { + try { + return chroma(hex).luminance() + } catch { + return 0 + } +} + +/** One background surface and every foreground token evaluated on it. */ +export interface ContrastPairing { + background: ColorToken + foregrounds: { token: ColorToken; verdict: ContrastVerdict }[] +} + +// Tokens that can plausibly carry text / icons and are worth checking as a +// foreground on a surface. +const FOREGROUND_CATEGORIES: ColorCategory[] = ['text', 'primary', 'secondary', 'accent', 'neutral'] + +/** + * Choose the surfaces to evaluate foregrounds against. Prefers tokens the + * scanner classified as `background`; when a scan yields none, falls back to the + * luminance extremes (the lightest — and, if distinct, the darkest — token) so + * the report still covers the likely page surfaces instead of coming up empty. + */ +function pickBackgrounds(colors: ColorToken[]): ColorToken[] { + const explicit = colors.filter((c) => c.category === 'background') + if (explicit.length > 0) return explicit + if (colors.length === 0) return [] + + const byLuminance = [...colors].sort((a, b) => luminance(b.hex) - luminance(a.hex)) + const picks: ColorToken[] = [byLuminance[0]] + const darkest = byLuminance[byLuminance.length - 1] + if (darkest.hex.toLowerCase() !== picks[0].hex.toLowerCase()) picks.push(darkest) + return picks +} + +/** + * Build the contrast report for the extracted color tokens: for each background + * surface, every relevant foreground token scored against it, best contrast + * first. A foreground identical to its background is skipped, and backgrounds + * that end up with no foreground are dropped. + */ +export function buildContrastReport(colors: ColorToken[]): ContrastPairing[] { + const backgrounds = pickBackgrounds(colors) + + return backgrounds + .map((background) => { + const candidates = colors.filter((c) => FOREGROUND_CATEGORIES.includes(c.category)) + // If categorisation left us with nothing usable, fall back to every token. + const pool = candidates.length > 0 ? candidates : colors + + const foregrounds = pool + .filter((token) => token.hex.toLowerCase() !== background.hex.toLowerCase()) + .map((token) => ({ token, verdict: evaluateContrast(token.hex, background.hex) })) + .sort((a, b) => b.verdict.ratio - a.verdict.ratio) + + return { background, foregrounds } + }) + .filter((pairing) => pairing.foregrounds.length > 0) +} diff --git a/src/lib/crawl-zip.ts b/src/lib/crawl-zip.ts new file mode 100644 index 0000000..1cec50d --- /dev/null +++ b/src/lib/crawl-zip.ts @@ -0,0 +1,127 @@ +// PixelLens — Crawl → multi-file ZIP export +// +// Alternative to the single concatenated .md document produced by the crawler: +// packages each crawled page as its own Markdown file plus a linking `index.md`, +// all bundled into a .zip via fflate. Purely additive — the single-file path in +// CrawlView is untouched. The page bodies come straight from `CrawlResult.pages` +// (already exposed by the crawler), so no crawler change is needed to zip them. + +import { zipSync, strToU8 } from 'fflate' +import type { CrawlResult, CrawlPageResult } from '@/types/crawl' + +/** Folder holding the individual page files inside the archive. */ +const PAGES_DIR = 'pages' + +/** Slugify a title into a filesystem-safe stem (mirrors the crawler's anchors). */ +function slugify(value: string): string { + return value + .normalize('NFKD') + .replace(/[\u0300-\u036f]/g, '') + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-+|-+$/g, '') + .slice(0, 60) + .replace(/-+$/g, '') +} + +/** YAML double-quoted scalar: escape `\` and `"`, flatten newlines. */ +function yamlString(value: string): string { + const escaped = value + .replace(/\\/g, '\\\\') + .replace(/"/g, '\\"') + .replace(/[\r\n]+/g, ' ') + .trim() + return `"${escaped}"` +} + +/** Per-page Markdown file: minimal frontmatter + title + URL + body. */ +function pageFile(page: CrawlPageResult): string { + const frontmatter = [ + '---', + `title: ${yamlString(page.title || 'Untitled')}`, + `url: ${yamlString(page.url)}`, + `words: ${page.wordCount}`, + '---', + ].join('\n') + return [frontmatter, '', `# ${page.title || 'Untitled'}`, '', `\`${page.url}\``, '', page.markdown.trim(), ''].join('\n') +} + +/** `index.md`: global frontmatter + a table of contents linking each page file. */ +function indexFile(result: CrawlResult, entries: { page: CrawlPageResult; path: string }[]): string { + const { host, startUrl, stats, generatedAt } = result + const frontmatter = [ + '---', + `site: ${yamlString(host)}`, + `url: ${yamlString(startUrl)}`, + `crawledAt: ${yamlString(generatedAt)}`, + `pages: ${entries.length}`, + `skipped: ${stats.skippedCount}`, + `discovery: ${yamlString(stats.discovery)}`, + 'generator: "PixelLens"', + '---', + ].join('\n') + + const header = [ + `# ${host} — full site`, + '', + `> ${entries.length} page${entries.length === 1 ? '' : 's'} crawled` + + (stats.skippedCount > 0 ? ` · ${stats.skippedCount} skipped` : '') + + ` · discovered via ${stats.discovery}`, + ].join('\n') + + const toc = [ + '## Pages', + '', + ...entries.map((e, i) => `${i + 1}. [${e.page.title || 'Untitled'}](${e.path}) — ${e.page.url}`), + ].join('\n') + + return [frontmatter, '', header, '', toc, ''].join('\n') +} + +/** + * Build the archive's file map: `index.md` at the root plus one + * `pages/.md` per crawled page. Slugs are de-duplicated with a `-N` + * suffix so no two pages collide on the same filename. Exposed (not just the + * zipped bytes) so the structure can be asserted directly in tests. + */ +export function buildZipEntries(result: CrawlResult): Record { + const seen = new Map() + const entries = result.pages.map((page, i) => { + const base = slugify(page.title) || `page-${i + 1}` + const count = seen.get(base) ?? 0 + seen.set(base, count + 1) + const stem = count === 0 ? base : `${base}-${count}` + return { page, path: `${PAGES_DIR}/${stem}.md` } + }) + + const files: Record = { 'index.md': indexFile(result, entries) } + for (const { page, path } of entries) files[path] = pageFile(page) + return files +} + +/** Zip the crawl result into `.zip` bytes: one Markdown file per page + index. */ +export function buildCrawlZip(result: CrawlResult): Uint8Array { + const files = buildZipEntries(result) + const zippable: Record = {} + for (const [path, content] of Object.entries(files)) zippable[path] = strToU8(content) + return zipSync(zippable, { level: 6 }) +} + +/** A friendly `.zip` filename for a crawl result, e.g. `example.com-site.zip`. */ +export function crawlZipFilename(result: CrawlResult): string { + return `${result.host}-site.zip` +} + +/** Build the zip and trigger a browser download (blob URL + transient anchor). */ +export function downloadCrawlZip(result: CrawlResult): void { + const bytes = buildCrawlZip(result) + // Copy into a fresh ArrayBuffer-backed view so Blob gets a plain ArrayBuffer + // (not fflate's possibly pooled/shared buffer). + const blob = new Blob([bytes.slice()], { type: 'application/zip' }) + const url = URL.createObjectURL(blob) + const a = document.createElement('a') + a.href = url + a.download = crawlZipFilename(result) + a.click() + URL.revokeObjectURL(url) +} diff --git a/src/lib/storage.ts b/src/lib/storage.ts index 8e136d8..ec9cd41 100644 --- a/src/lib/storage.ts +++ b/src/lib/storage.ts @@ -80,6 +80,7 @@ export type PanelViewMode = | 'inspect' | 'scan' | 'design-system' + | 'contrast' | 'export' | 'history' | 'markdown' diff --git a/src/sidepanel/App.tsx b/src/sidepanel/App.tsx index e015ab7..b5970f7 100644 --- a/src/sidepanel/App.tsx +++ b/src/sidepanel/App.tsx @@ -8,6 +8,7 @@ import { MarkdownLogo, GlobeHemisphereWest, GearSix, + CircleHalf, } from '@phosphor-icons/react' import gsap from 'gsap' import { usePanelStore, type PanelMode } from './store' @@ -20,9 +21,15 @@ import ExportView from './views/ExportView' import HistoryView from './views/HistoryView' import MarkdownView from './views/MarkdownView' import CrawlView from './views/CrawlView' +import ContrastCheckerView from './views/ContrastCheckerView' import SettingsView from './views/SettingsView' import PixelLensLogo from './components/PixelLensLogo' -import CommandPalette, { createNavigationCommands, isMacPlatform } from './components/CommandPalette' +import CommandPalette, { + createNavigationCommands, + createContrastCommands, + createCrawlZipCommands, + isMacPlatform, +} from './components/CommandPalette' import { prefersReducedMotion } from './reducedMotion' const MAIN_TABS: { mode: PanelMode; label: string; icon: typeof MagnifyingGlass }[] = [ @@ -32,6 +39,7 @@ const MAIN_TABS: { mode: PanelMode; label: string; icon: typeof MagnifyingGlass ] const FOOTER_TABS: { mode: PanelMode; icon: typeof Export; label: string }[] = [ + { mode: 'contrast', icon: CircleHalf, label: 'Contrast checker' }, { mode: 'export', icon: Export, label: 'Export' }, { mode: 'markdown', icon: MarkdownLogo, label: 'Markdown' }, { mode: 'crawl', icon: GlobeHemisphereWest, label: 'Crawl site' }, @@ -51,6 +59,7 @@ function App() { const setCrawlProgress = usePanelStore((s) => s.setCrawlProgress) const setCrawlResult = usePanelStore((s) => s.setCrawlResult) const setCrawlRunning = usePanelStore((s) => s.setCrawlRunning) + const setCrawlExportMode = usePanelStore((s) => s.setCrawlExportMode) const tabsRef = useRef<(HTMLButtonElement | null)[]>([]) const indicatorRef = useRef(null) @@ -59,7 +68,14 @@ function App() { // Command palette: transient, LOCAL state only — it is an accelerator, never // part of the persisted panel envelope. const [paletteOpen, setPaletteOpen] = useState(false) - const commands = useMemo(() => createNavigationCommands(setMode), [setMode]) + const commands = useMemo( + () => [ + ...createNavigationCommands(setMode), + ...createContrastCommands(setMode), + ...createCrawlZipCommands(setMode, setCrawlExportMode), + ], + [setMode, setCrawlExportMode], + ) const shortcutLabel = isMacPlatform() ? '⌘K' : 'Ctrl K' // Global ⌘K / Ctrl+K toggles the palette from anywhere in the panel. @@ -228,6 +244,8 @@ function App() { return case 'design-system': return + case 'contrast': + return case 'export': return case 'history': diff --git a/src/sidepanel/components/CommandPalette.tsx b/src/sidepanel/components/CommandPalette.tsx index ef3bfed..af57b89 100644 --- a/src/sidepanel/components/CommandPalette.tsx +++ b/src/sidepanel/components/CommandPalette.tsx @@ -8,9 +8,11 @@ import { MarkdownLogo, GlobeHemisphereWest, GearSix, + CircleHalf, + FileZip, type Icon, } from '@phosphor-icons/react' -import type { PanelMode } from '../store' +import type { PanelMode, CrawlExportMode } from '../store' import { prefersReducedMotion } from '../reducedMotion' /** @@ -71,6 +73,47 @@ export function createNavigationCommands(setMode: (mode: PanelMode) => void): Co ] } +/** + * Contrast checker destination (Analyze group). Split out as its own factory so + * it composes with the base registry via a spread in App.tsx. + */ +export function createContrastCommands(setMode: (mode: PanelMode) => void): Command[] { + return [ + { + id: 'nav-contrast', + label: 'Contrast checker', + icon: CircleHalf, + group: 'Analyze', + keywords: ['wcag', 'a11y', 'accessibility', 'ratio', 'aa', 'aaa', 'readability', 'contrast'], + run: () => setMode('contrast'), + }, + ] +} + +/** + * "Crawl site → ZIP" accelerator (Convert group): presets the crawl export shape + * to a multi-file ZIP and jumps to the Crawl view. It does not auto-start the + * network crawl — the user still triggers it — so no surprise traffic fires. + */ +export function createCrawlZipCommands( + setMode: (mode: PanelMode) => void, + setCrawlExportMode: (mode: CrawlExportMode) => void, +): Command[] { + return [ + { + id: 'crawl-zip', + label: 'Crawl site → ZIP', + icon: FileZip, + group: 'Convert', + keywords: ['zip', 'archive', 'multi-file', 'pages', 'export', 'download', 'crawl'], + run: () => { + setCrawlExportMode('zip') + setMode('crawl') + }, + }, + ] +} + /** True on macOS/iOS so the shortcut hint shows the Command key instead of Ctrl. */ export function isMacPlatform(): boolean { if (typeof navigator === 'undefined') return false diff --git a/src/sidepanel/store.ts b/src/sidepanel/store.ts index 13ea079..b404f64 100644 --- a/src/sidepanel/store.ts +++ b/src/sidepanel/store.ts @@ -10,12 +10,16 @@ export type PanelMode = | 'inspect' | 'scan' | 'design-system' + | 'contrast' | 'export' | 'history' | 'markdown' | 'crawl' | 'settings' +/** How a finished crawl is exported: one concatenated .md, or a multi-file .zip. */ +export type CrawlExportMode = 'single' | 'zip' + export interface ScanProgress { percent: number phase: string @@ -38,6 +42,9 @@ interface PanelState { crawlProgress: CrawlProgress | null crawlResult: CrawlResult | null crawlError: string | null + // Chosen crawl export shape. Transient (never persisted): a fresh panel always + // defaults to the single-file .md; the ⌘K "Crawl → ZIP" command presets 'zip'. + crawlExportMode: CrawlExportMode // True once `persist` has finished its async rehydration from // chrome.storage.local. The panel UI gates on this so it never paints the // default state first and then snaps to the restored one (see App.tsx). @@ -60,6 +67,7 @@ interface PanelState { setCrawlProgress: (progress: CrawlProgress | null) => void setCrawlResult: (result: CrawlResult | null) => void setCrawlError: (error: string | null) => void + setCrawlExportMode: (mode: CrawlExportMode) => void setHasHydrated: (hydrated: boolean) => void } @@ -80,6 +88,7 @@ export const usePanelStore = create()( crawlProgress: null, crawlResult: null, crawlError: null, + crawlExportMode: 'single', hasHydrated: false, setMode: (mode) => set({ activeMode: mode }), @@ -99,6 +108,7 @@ export const usePanelStore = create()( setCrawlProgress: (progress) => set({ crawlProgress: progress }), setCrawlResult: (result) => set({ crawlResult: result }), setCrawlError: (error) => set({ crawlError: error }), + setCrawlExportMode: (mode) => set({ crawlExportMode: mode }), setHasHydrated: (hydrated) => set({ hasHydrated: hydrated }), }), { diff --git a/src/sidepanel/views/ContrastCheckerView.tsx b/src/sidepanel/views/ContrastCheckerView.tsx new file mode 100644 index 0000000..82b972c --- /dev/null +++ b/src/sidepanel/views/ContrastCheckerView.tsx @@ -0,0 +1,198 @@ +import { useEffect, useMemo, useRef } from 'react' +import { CircleHalf, Scan } from '@phosphor-icons/react' +import gsap from 'gsap' +import { usePanelStore } from '../store' +import { prefersReducedMotion } from '../reducedMotion' +import { buildContrastReport, formatRatio, type ContrastLevel } from '@/lib/contrast' + +// Pill palette. Each level carries its own label text, so the verdict never +// relies on color alone (WCAG SC 1.4.1). Green = AAA, amber = AA-only, red = fail. +const LEVEL_STYLE: Record = { + AAA: { label: 'AAA', className: 'border-[#22C55E]/30 bg-[#22C55E]/12 text-[#4ADE80]' }, + AA: { label: 'AA', className: 'border-[#F59E0B]/30 bg-[#F59E0B]/12 text-[#FBBF24]' }, + fail: { label: 'Fail', className: 'border-[#EF4444]/30 bg-[#EF4444]/12 text-[#F87171]' }, +} + +function VerdictPill({ scope, level, ratio }: { scope: 'Normal' | 'Large'; level: ContrastLevel; ratio: number }) { + const s = LEVEL_STYLE[level] + const verdict = level === 'fail' ? 'fails WCAG' : `passes ${level}` + return ( + + {scope} + + {s.label} + + ) +} + +function ContrastCheckerView() { + const designSystem = usePanelStore((s) => s.designSystem) + const setMode = usePanelStore((s) => s.setMode) + const listRef = useRef(null) + + const report = useMemo( + () => (designSystem ? buildContrastReport(designSystem.colors) : []), + [designSystem], + ) + + // Flatten every evaluated pair once for the summary strip. + const summary = useMemo(() => { + let total = 0 + let passAA = 0 + for (const pairing of report) { + for (const fg of pairing.foregrounds) { + total++ + if (fg.verdict.aaNormal) passAA++ + } + } + return { total, passAA } + }, [report]) + + // Staggered reveal of the background sections. Honors reduced motion by + // snapping to the end state (no tween) via the shared helper. + useEffect(() => { + const root = listRef.current + if (!root || report.length === 0) return + const sections = root.querySelectorAll('[data-contrast-section]') + if (sections.length === 0) return + if (prefersReducedMotion()) { + gsap.set(sections, { opacity: 1, y: 0 }) + return + } + const tween = gsap.fromTo( + sections, + { opacity: 0, y: 8 }, + { opacity: 1, y: 0, duration: 0.35, ease: 'power2.out', stagger: 0.06 }, + ) + return () => { + tween.kill() + } + }, [report]) + + // --- Empty — no scan yet ---------------------------------------------------------- + if (!designSystem) { + return ( +
+
+ +
+
+

Check color contrast

+

+ Scan a page first, then see which of its extracted colors meet WCAG AA / AAA + contrast on each background. +

+ +
+
+ ) + } + + // --- No usable colors ------------------------------------------------------------- + if (report.length === 0) { + return ( +
+
+ +
+
+

Not enough colors

+

+ This scan didn’t yield enough distinct colors to evaluate contrast. Try + scanning a richer page. +

+
+
+ ) + } + + // --- Results ---------------------------------------------------------------------- + return ( +
+ {/* Meta header + summary */} +
+
+ +

+ Contrast · {designSystem.metadata.title || designSystem.metadata.url} +

+
+

+ {summary.passAA}/ + {summary.total} pairs pass AA for normal text +

+
+ + {/* Pairings */} +
+ {report.map((pairing) => ( +
+ {/* Background surface header */} +
+
+ + {/* Foreground rows */} +
    + {pairing.foregrounds.map((fg) => ( +
  • +
    + +
    +

    {fg.token.hex}

    +

    + {fg.token.category} +

    +
    +
    + + {formatRatio(fg.verdict.ratio)} + + :1 +
    +
    +
    + + +
    +
    • + ))} +
+
+ ))} +
+
+ ) +} + +export default ContrastCheckerView diff --git a/src/sidepanel/views/CrawlView.tsx b/src/sidepanel/views/CrawlView.tsx index 842a7d6..ed3c0ec 100644 --- a/src/sidepanel/views/CrawlView.tsx +++ b/src/sidepanel/views/CrawlView.tsx @@ -8,6 +8,7 @@ import { Stop, ArrowClockwise, Files, + FileZip, Prohibit, Database, TreeStructure, @@ -16,6 +17,7 @@ import { usePanelStore } from '../store' import { sendMessage } from '@/lib/messaging' import { MessageType } from '@/types/messages' import { copyToClipboard, downloadMarkdown } from '@/lib/export' +import { downloadCrawlZip } from '@/lib/crawl-zip' import MarkdownPreview from '../components/MarkdownPreview' // Le download contient TOUT le document ; la preview est tronquée au-delà de ce seuil @@ -37,6 +39,8 @@ function CrawlView() { const setProgress = usePanelStore((s) => s.setCrawlProgress) const setResult = usePanelStore((s) => s.setCrawlResult) const setError = usePanelStore((s) => s.setCrawlError) + const exportMode = usePanelStore((s) => s.crawlExportMode) + const setExportMode = usePanelStore((s) => s.setCrawlExportMode) const [copied, setCopied] = useState(false) const handleStart = useCallback(async () => { @@ -193,7 +197,12 @@ function CrawlView() { } const handleDownload = () => { - downloadMarkdown(`${result.host}-site.md`, result.document) + if (exportMode === 'zip') { + // Multi-file archive: one .md per page + a linking index.md. + downloadCrawlZip(result) + } else { + downloadMarkdown(`${result.host}-site.md`, result.document) + } } return ( @@ -253,28 +262,61 @@ function CrawlView() {
{/* Actions */} -
- - + {([ + { value: 'single', label: '1 file', icon: Files }, + { value: 'zip', label: 'ZIP', icon: FileZip }, + ] as const).map(({ value, label, icon: Icon }) => { + const isActive = exportMode === value + return ( + + ) + })} +
+ +
+ + +
) From 8e14ab62b0e5f1e63ddf4a4d8fc75307ae2e87fa Mon Sep 17 00:00:00 2001 From: Adam Date: Wed, 1 Jul 2026 04:18:45 +0200 Subject: [PATCH 4/5] fix(crawl): surface skip reasons + robots.txt toggle + same-site - Each skipped URL now carries a reason (http-, timeout, non-text, cross-origin, network, robots). CrawlView shows a breakdown, so 'everything skipped' is no longer silent. - fetchTextWithTimeout returns a discriminated result capturing the real HTTP status (http-403 = anti-bot, etc). - 'Respect robots.txt' toggle (default on): the user is a human on their own session; turn off to include disallowed pages. Hint shown when pages are blocked. - same-site matching (apex <-> www) instead of strict same-origin, so www-hosted sitemaps are no longer excluded. - Unit tests 209 -> 218. --- src/lib/__tests__/crawl-fetch.test.ts | 39 ++++--- src/lib/__tests__/crawl-zip.test.ts | 1 + src/lib/__tests__/crawler.test.ts | 141 +++++++++++++++++++++++++- src/lib/crawl-fetch.ts | 34 +++++-- src/lib/crawler.ts | 102 ++++++++++++++----- src/sidepanel/store.ts | 7 ++ src/sidepanel/views/CrawlView.tsx | 78 +++++++++++++- src/types/crawl.ts | 49 ++++++++- 8 files changed, 388 insertions(+), 63 deletions(-) diff --git a/src/lib/__tests__/crawl-fetch.test.ts b/src/lib/__tests__/crawl-fetch.test.ts index a814848..3109d92 100644 --- a/src/lib/__tests__/crawl-fetch.test.ts +++ b/src/lib/__tests__/crawl-fetch.test.ts @@ -1,10 +1,11 @@ import { describe, it, expect, vi, afterEach } from 'vitest' import { fetchTextWithTimeout, DEFAULT_FETCH_TIMEOUT_MS } from '../crawl-fetch' -// Reponse minimale facon `Response` pour piloter ok / content-type / body. -function fakeResponse(body: string, ct = 'text/html', ok = true) { +// Reponse minimale facon `Response` pour piloter ok / status / content-type / body. +function fakeResponse(body: string, ct = 'text/html', ok = true, status = ok ? 200 : 500) { return { ok, + status, headers: { get: (h: string) => (h.toLowerCase() === 'content-type' ? ct : null) }, text: () => Promise.resolve(body), } as unknown as Response @@ -28,35 +29,43 @@ afterEach(() => { }) describe('fetchTextWithTimeout', () => { - it('renvoie le corps texte sur une reponse OK et textuelle', async () => { + it('renvoie { ok:true, text } sur une reponse OK et textuelle', async () => { const fetchImpl = vi.fn(() => Promise.resolve(fakeResponse('hi'))) const res = await fetchTextWithTimeout('https://ex.com/', { fetch: fetchImpl }) - expect(res).toBe('hi') + expect(res).toEqual({ ok: true, text: 'hi' }) // Le signal transmis a fetch n'a pas ete aborte sur un succes. expect(fetchImpl.mock.calls[0][1]?.signal?.aborted).toBe(false) }) - it('renvoie null sur reponse non-OK', async () => { - const fetchImpl = vi.fn(() => Promise.resolve(fakeResponse('nope', 'text/html', false))) - expect(await fetchTextWithTimeout('https://ex.com/', { fetch: fetchImpl })).toBeNull() + it('renvoie la raison http- reelle sur reponse non-OK (ex. anti-bot 403)', async () => { + const fetchImpl = vi.fn(() => + Promise.resolve(fakeResponse('forbidden', 'text/html', false, 403)), + ) + expect(await fetchTextWithTimeout('https://ex.com/', { fetch: fetchImpl })).toEqual({ + ok: false, + reason: 'http-403', + }) }) - it('renvoie null sur contenu non textuel (binaire)', async () => { + it('renvoie { ok:false, reason:non-text } sur contenu non textuel (binaire)', async () => { const fetchImpl = vi.fn(() => Promise.resolve(fakeResponse(' ', 'image/png'))) - expect(await fetchTextWithTimeout('https://ex.com/', { fetch: fetchImpl })).toBeNull() + expect(await fetchTextWithTimeout('https://ex.com/', { fetch: fetchImpl })).toEqual({ + ok: false, + reason: 'non-text', + }) }) - it('renvoie null et aborte la requete au depassement du timeout (serveur muet)', async () => { + it('renvoie reason:timeout et aborte la requete au depassement du timeout (serveur muet)', async () => { vi.useFakeTimers() const fetchImpl = hangingFetch() const p = fetchTextWithTimeout('https://ex.com/', { fetch: fetchImpl, timeoutMs: 1000 }) await vi.advanceTimersByTimeAsync(1000) - expect(await p).toBeNull() + expect(await p).toEqual({ ok: false, reason: 'timeout' }) // Le fetch en vol a bien recu l'abort declenche par le timeout. expect(fetchImpl.mock.calls[0][1]?.signal?.aborted).toBe(true) }) - it('coupe le fetch en vol quand le signal externe (Stop) est aborte', async () => { + it('coupe le fetch en vol et renvoie reason:network quand le signal externe (Stop) est aborte', async () => { const external = new AbortController() const fetchImpl = hangingFetch() const p = fetchTextWithTimeout('https://ex.com/', { @@ -66,17 +75,17 @@ describe('fetchTextWithTimeout', () => { timeoutMs: 999_999, }) external.abort() - expect(await p).toBeNull() + expect(await p).toEqual({ ok: false, reason: 'network' }) expect(fetchImpl.mock.calls[0][1]?.signal?.aborted).toBe(true) }) - it('renvoie null immediatement si le signal externe est deja aborte', async () => { + it('renvoie reason:network immediatement si le signal externe est deja aborte', async () => { const external = new AbortController() external.abort() const fetchImpl = hangingFetch() expect( await fetchTextWithTimeout('https://ex.com/', { fetch: fetchImpl, signal: external.signal }), - ).toBeNull() + ).toEqual({ ok: false, reason: 'network' }) }) it('expose un timeout par defaut de 15000 ms', () => { diff --git a/src/lib/__tests__/crawl-zip.test.ts b/src/lib/__tests__/crawl-zip.test.ts index 77f003c..aa9c6d0 100644 --- a/src/lib/__tests__/crawl-zip.test.ts +++ b/src/lib/__tests__/crawl-zip.test.ts @@ -16,6 +16,7 @@ function makeResult(): CrawlResult { { url: 'https://ex.com/c', title: '', markdown: 'body c', wordCount: 1 }, ], skipped: ['https://ex.com/private'], + skippedReasons: { robots: 1 }, document: 'IGNORED single-file document', generatedAt: '2026-07-01T00:00:00.000Z', stats: { pageCount: 3, skippedCount: 1, bytes: 100, discovery: 'crawl' }, diff --git a/src/lib/__tests__/crawler.test.ts b/src/lib/__tests__/crawler.test.ts index 5cc9371..ea2f729 100644 --- a/src/lib/__tests__/crawler.test.ts +++ b/src/lib/__tests__/crawler.test.ts @@ -2,6 +2,7 @@ import { describe, it, expect } from 'vitest' import { normalizeUrl, sameOrigin, + sameSite, parseRobots, robotsAllows, parseSitemap, @@ -11,16 +12,18 @@ import { type CrawlDeps, } from '../crawler' import { htmlDocumentToMarkdown } from '../markdown' -import type { CrawlPageResult } from '@/types/crawl' +import type { CrawlPageResult, FetchTextResult } from '@/types/crawl' // --- Helpers ------------------------------------------------------------------------ -// Fake fetch piloté par une table URL -> texte (null = échec / absent). +// Fake fetch piloté par une table URL -> texte (null = échec / absent). Adapte le +// texte au résultat discriminé : présent -> { ok:true, text }, absent -> échec réseau. function makeFetch(pages: Record) { const calls: string[] = [] - const fetchText = async (url: string): Promise => { + const fetchText = async (url: string): Promise => { calls.push(url) - return url in pages ? pages[url] : null + const text = url in pages ? pages[url] : null + return text != null ? { ok: true, text } : { ok: false, reason: 'network' } } return { fetchText, calls } } @@ -81,6 +84,26 @@ describe('sameOrigin', () => { }) }) +// --- sameSite (filtre de crawl, plus large : apex <-> www) -------------------------- + +describe('sameSite', () => { + it('vrai pour host exact', () => { + expect(sameSite('https://ex.com/a', 'https://ex.com')).toBe(true) + }) + it('accepte la variante apex <-> www du même domaine', () => { + expect(sameSite('https://www.ex.com/a', 'https://ex.com')).toBe(true) + expect(sameSite('https://ex.com/a', 'https://www.ex.com')).toBe(true) + }) + it('reste STRICT hors domaine (autre site, sous-domaine, CDN)', () => { + expect(sameSite('https://other.com/a', 'https://ex.com')).toBe(false) + expect(sameSite('https://blog.ex.com/a', 'https://ex.com')).toBe(false) + expect(sameSite('https://cdn.ex.com/a', 'https://www.ex.com')).toBe(false) + }) + it('faux pour protocole différent', () => { + expect(sameSite('http://ex.com/a', 'https://ex.com')).toBe(false) + }) +}) + // --- robots.txt --------------------------------------------------------------------- describe('parseRobots / robotsAllows', () => { @@ -386,6 +409,116 @@ describe('crawlSite — fallback BFS same-origin', () => { }) }) +// --- crawlSite : robots off / same-site / raisons de skip --------------------------- + +describe('crawlSite — respectRobots=false', () => { + it('inclut les chemins interdits et ne télécharge même pas robots.txt', async () => { + const { fetchText, calls } = makeFetch({ + 'https://ex.com/robots.txt': 'User-agent: *\nDisallow: /private\n', + 'https://ex.com/sitemap.xml': null, + 'https://ex.com/': page(['/public', '/private']), + 'https://ex.com/public': page([]), + 'https://ex.com/private': page([]), + }) + const res = await crawlSite( + { startUrl: 'https://ex.com/', respectRobots: false }, + baseDeps(fetchText), + ) + expect(sortedPageUrls(res.pages)).toEqual([ + 'https://ex.com/', + 'https://ex.com/private', + 'https://ex.com/public', + ]) + expect(calls).not.toContain('https://ex.com/robots.txt') + expect(res.skippedReasons.robots ?? 0).toBe(0) + }) +}) + +describe('crawlSite — découverte same-site (apex <-> www)', () => { + it('accepte les URLs www du sitemap quand la page de départ est sur l’apex', async () => { + const { fetchText } = makeFetch({ + 'https://ex.com/robots.txt': null, + 'https://ex.com/sitemap.xml': + 'https://www.ex.com/a' + + 'https://www.ex.com/b', + 'https://ex.com/': page([]), + 'https://www.ex.com/a': page([]), + 'https://www.ex.com/b': page([]), + }) + const res = await crawlSite({ startUrl: 'https://ex.com/' }, baseDeps(fetchText)) + expect(res.stats.discovery).toBe('sitemap') + expect(sortedPageUrls(res.pages)).toEqual([ + 'https://ex.com/', + 'https://www.ex.com/a', + 'https://www.ex.com/b', + ]) + }) + + it('rejette toujours un autre domaine / sous-domaine (hors apex/www)', async () => { + const { fetchText } = makeFetch({ + 'https://ex.com/robots.txt': null, + 'https://ex.com/sitemap.xml': null, + 'https://ex.com/': page(['https://cdn.ex.com/x', 'https://other.com/y', '/ok']), + 'https://ex.com/ok': page([]), + }) + const res = await crawlSite({ startUrl: 'https://ex.com/' }, baseDeps(fetchText)) + expect(sortedPageUrls(res.pages)).toEqual(['https://ex.com/', 'https://ex.com/ok']) + }) +}) + +describe('crawlSite — breakdown des raisons de skip', () => { + it('agrège chaque skip par raison (robots / http-403 / non-text / empty)', async () => { + const table: Record = { + 'https://ex.com/robots.txt': { ok: true, text: 'User-agent: *\nDisallow: /blocked\n' }, + 'https://ex.com/sitemap.xml': { ok: false, reason: 'network' }, + 'https://ex.com/': { + ok: true, + text: page(['/blocked', '/forbidden', '/asset', '/empty', '/ok']), + }, + 'https://ex.com/forbidden': { ok: false, reason: 'http-403' }, + 'https://ex.com/asset': { ok: false, reason: 'non-text' }, + 'https://ex.com/empty': { ok: true, text: page([]) }, + 'https://ex.com/ok': { ok: true, text: page([]) }, + } + const fetchText = async (url: string): Promise => + url in table ? table[url] : { ok: false, reason: 'network' } + // Convert renvoie du Markdown vide pour /empty -> compté « empty ». + const convert = (_html: string, url: string): CrawlPageResult | null => + url === 'https://ex.com/empty' + ? { url, title: 'Empty', markdown: ' ', wordCount: 0 } + : { url, title: `T ${url}`, markdown: `Body ${url}`, wordCount: 2 } + + const res = await crawlSite( + { startUrl: 'https://ex.com/' }, + { fetchText, convert, delay: () => Promise.resolve() }, + ) + expect(sortedPageUrls(res.pages)).toEqual(['https://ex.com/', 'https://ex.com/ok']) + expect(res.skippedReasons).toEqual({ + robots: 1, + 'http-403': 1, + 'non-text': 1, + empty: 1, + }) + expect(res.stats.skippedCount).toBe(4) + }) + + it('porte la dernière raison de skip dans la progression', async () => { + const reasons: (string | undefined)[] = [] + const { fetchText } = makeFetch({ + 'https://ex.com/robots.txt': 'User-agent: *\nDisallow: /blocked\n', + 'https://ex.com/sitemap.xml': null, + 'https://ex.com/': page(['/blocked', '/ok']), + 'https://ex.com/ok': page([]), + }) + await crawlSite( + { startUrl: 'https://ex.com/' }, + baseDeps(fetchText, { onProgress: (p) => reasons.push(p.lastSkipReason) }), + ) + // Après le skip robots de /blocked, un emit ultérieur porte la raison. + expect(reasons).toContain('robots') + }) +}) + // --- Réutilisation du moteur Markdown sur un Document fetché ------------------------- describe('htmlDocumentToMarkdown (page fetchée via DOMParser)', () => { diff --git a/src/lib/crawl-fetch.ts b/src/lib/crawl-fetch.ts index 94fb4a8..151d4e0 100644 --- a/src/lib/crawl-fetch.ts +++ b/src/lib/crawl-fetch.ts @@ -4,8 +4,12 @@ // - a un timeout dur (un serveur muet ne doit pas figer le crawl à vie) ; // - écoute un signal d'annulation externe (Stop) pour couper le fetch EN VOL, // au lieu de n'agir qu'entre deux pages. -// Renvoie `null` sur timeout / annulation / erreur réseau / contenu non textuel : -// l'orchestrateur (lib/crawler) compte alors la page comme « skipped » et poursuit. +// Renvoie un RÉSULTAT DISCRIMINÉ ({ ok:true, text } | { ok:false, reason }) au lieu +// d'un `null` opaque : le status HTTP réel (ex. 403 anti-bot), le timeout, le binaire +// et l'erreur réseau/CORS deviennent des RAISONS explicites. L'orchestrateur +// (lib/crawler) agrège ces raisons pour dire à l'utilisateur POURQUOI ça skippe. + +import type { FetchTextResult } from '@/types/crawl' /** Timeout par défaut d'une requête de crawl (ms). */ export const DEFAULT_FETCH_TIMEOUT_MS = 15_000 @@ -21,13 +25,14 @@ export interface FetchTextOptions { /** * Récupère le texte d'une URL avec timeout + annulation. Ne lit que du contenu - * textuel (text/html/xml/json/plain). Renvoie `null` sur réponse non-OK, contenu - * binaire, timeout, abort (Stop) ou erreur réseau — jamais de rejet propagé. + * textuel (text/html/xml/json/plain). Ne rejette jamais : renvoie un résultat + * discriminé — `{ ok:true, text }` en cas de succès, sinon `{ ok:false, reason }` + * avec la raison réelle (`http-`, `timeout`, `non-text`, `network`). */ export async function fetchTextWithTimeout( url: string, options: FetchTextOptions = {}, -): Promise { +): Promise { const doFetch = options.fetch ?? fetch const timeoutMs = options.timeoutMs ?? DEFAULT_FETCH_TIMEOUT_MS @@ -40,7 +45,13 @@ export async function fetchTextWithTimeout( if (external.aborted) controller.abort() else external.addEventListener('abort', onExternalAbort) } - const timer = setTimeout(() => controller.abort(), timeoutMs) + // Distingue un abort déclenché par le timeout (raison `timeout`) d'une autre + // rupture (Stop externe / erreur réseau / CORS → raison `network`). + let timedOut = false + const timer = setTimeout(() => { + timedOut = true + controller.abort() + }, timeoutMs) try { const res = await doFetch(url, { @@ -48,14 +59,15 @@ export async function fetchTextWithTimeout( redirect: 'follow', signal: controller.signal, }) - if (!res.ok) return null + // Status HTTP réel exposé comme raison (ex. `http-403` = anti-bot, `http-404`). + if (!res.ok) return { ok: false, reason: `http-${res.status}` } const ct = res.headers.get('content-type') ?? '' // Évite le binaire : on ne lit que du texte/HTML/XML/JSON. - if (ct && !/(text|html|xml|json|plain)/i.test(ct)) return null - return await res.text() + if (ct && !/(text|html|xml|json|plain)/i.test(ct)) return { ok: false, reason: 'non-text' } + return { ok: true, text: await res.text() } } catch { - // Timeout, abort (Stop), erreur réseau, CORS… → page ignorée, crawl poursuivi. - return null + // Timeout → `timeout` ; abort (Stop) / erreur réseau / CORS → `network`. + return { ok: false, reason: timedOut ? 'timeout' : 'network' } } finally { clearTimeout(timer) external?.removeEventListener('abort', onExternalAbort) diff --git a/src/lib/crawler.ts b/src/lib/crawler.ts index d6c3ef5..464e6e8 100644 --- a/src/lib/crawler.ts +++ b/src/lib/crawler.ts @@ -24,6 +24,9 @@ import type { CrawlPageResult, CrawlProgress, CrawlResult, + FetchTextResult, + SkipReason, + SkipReasonCounts, } from '@/types/crawl' /** Bornes par défaut (figées par le produit). */ @@ -34,8 +37,12 @@ const MAX_SUBSITEMAPS = 50 /** I/O et conversion injectés — permettent de tester l'orchestration sans réseau. */ export interface CrawlDeps { - /** Récupère le texte d'une URL ; `null` si échec / non-OK / non-textuel. */ - fetchText: (url: string) => Promise + /** + * Récupère le texte d'une URL. Résultat DISCRIMINÉ : `{ ok:true, text }` ou + * `{ ok:false, reason }` — la raison (http-/timeout/non-text/network) + * remonte jusqu'au breakdown affiché à l'utilisateur. + */ + fetchText: (url: string) => Promise /** Convertit le HTML d'une page en Markdown ; `null` si rien d'exploitable. */ convert: (html: string, url: string) => CrawlPageResult | null onProgress?: (progress: CrawlProgress) => void @@ -77,6 +84,29 @@ export function sameOrigin(url: string, origin: string): boolean { } } +/** + * Même SITE (filtre de crawl, plus large que sameOrigin) : même protocole ET même + * host, OU la variante apex↔www du même domaine (`example.com` ↔ `www.example.com`). + * On strippe UNIQUEMENT le `www.` de tête : reste STRICT sur tout autre sous-domaine + * (`blog.` / `cdn.` / autre domaine → refusés), donc pas de saut vers un autre site. + * + * Motivation : les sitemaps (et les liens canoniques) listent souvent l'hôte `www` + * alors que la page de départ est sur l'apex (ou l'inverse). Avec sameOrigin STRICT, + * ces URLs étaient toutes rejetées → 0 page découverte. NOTE : un fetch vers l'autre + * variante d'hôte peut rester bloqué par CORS depuis le content script ; ce n'est plus + * silencieux — il ressort désormais avec la raison `network` dans le breakdown. + */ +export function sameSite(url: string, origin: string): boolean { + try { + const u = new URL(url) + const o = new URL(origin) + if (u.protocol !== o.protocol) return false + return u.hostname.replace(/^www\./, '') === o.hostname.replace(/^www\./, '') + } catch { + return false + } +} + function pathWithQuery(url: string): string { try { const u = new URL(url) @@ -296,6 +326,7 @@ export async function crawlSite(options: CrawlOptions, deps: CrawlDeps): Promise const maxPages = options.maxPages ?? CRAWL_DEFAULTS.maxPages const maxDepth = options.maxDepth ?? CRAWL_DEFAULTS.maxDepth const delayMs = options.delayMs ?? CRAWL_DEFAULTS.delayMs + const respectRobots = options.respectRobots ?? true const onProgress = deps.onProgress ?? (() => {}) const shouldStop = deps.shouldStop ?? (() => false) const delay = deps.delay ?? realDelay @@ -304,43 +335,59 @@ export async function crawlSite(options: CrawlOptions, deps: CrawlDeps): Promise const origin = safeOrigin(options.startUrl) const host = safeHost(options.startUrl) + // deps.fetchText ne rejette pas en pratique, mais on blinde : un rejet inattendu + // devient une raison `network` explicite plutôt qu'un crash silencieux du crawl. + const fetchTextSafe = (url: string): Promise => + deps.fetchText(url).catch((): FetchTextResult => ({ ok: false, reason: 'network' })) + // robots.txt (best-effort : un échec n'interrompt pas le crawl, il autorise tout). - const robotsTxt = await deps.fetchText(origin + '/robots.txt').catch(() => null) - const disallow = robotsTxt ? parseRobots(robotsTxt) : [] + // Désactivable via respectRobots : à false on NE fetch même pas robots.txt. + let disallow: string[] = [] + if (respectRobots) { + const robotsRes = await fetchTextSafe(origin + '/robots.txt') + if (robotsRes.ok) disallow = parseRobots(robotsRes.text) + } const pages: CrawlPageResult[] = [] const skipped: string[] = [] + const skippedReasons: SkipReasonCounts = {} const visited = new Set() const counters = { skipped: 0 } + // Dernière raison de skip, portée au prochain emit() pour un feedback live. + let lastSkipReason: SkipReason | undefined const emit = (currentUrl: string, total: number): void => - onProgress({ done: pages.length, total, currentUrl, skipped: counters.skipped }) + onProgress({ done: pages.length, total, currentUrl, skipped: counters.skipped, lastSkipReason }) - const skip = (url: string): void => { + const skip = (url: string, reason: SkipReason): void => { counters.skipped++ skipped.push(url) + skippedReasons[reason] = (skippedReasons[reason] ?? 0) + 1 + lastSkipReason = reason } - // Traite une URL : garde same-origin + robots, fetch, convert. Renvoie le HTML - // (pour l'expansion de liens en BFS) ou null si la page a été ignorée. + // Traite une URL : garde same-site + robots, fetch, convert. Renvoie le HTML + // (pour l'expansion de liens en BFS) ou null si la page a été ignorée — avec, dans + // ce cas, une RAISON explicite enregistrée dans skippedReasons. const processOne = async (url: string, total: number): Promise => { - if (!sameOrigin(url, origin)) { - skip(url) + if (!sameSite(url, origin)) { + skip(url, 'cross-origin') return null } - if (!robotsAllows(pathWithQuery(url), disallow)) { - skip(url) + if (respectRobots && !robotsAllows(pathWithQuery(url), disallow)) { + skip(url, 'robots') return null } emit(url, total) - const html = await deps.fetchText(url).catch(() => null) - if (!html) { - skip(url) + const res = await fetchTextSafe(url) + if (!res.ok) { + skip(url, res.reason) return null } + const html = res.text const page = deps.convert(html, url) if (!page || !page.markdown.trim()) { - skip(url) + skip(url, 'empty') return null } pages.push(page) @@ -352,9 +399,9 @@ export async function crawlSite(options: CrawlOptions, deps: CrawlDeps): Promise // --- Découverte : sitemap.xml d'abord --- let discovery: CrawlDiscovery = 'crawl' let sitemapSeeds: string[] = [] - const sitemapXml = await deps.fetchText(origin + '/sitemap.xml').catch(() => null) - if (sitemapXml) { - sitemapSeeds = await collectSitemapUrls(sitemapXml, origin, deps, maxPages, delay, delayMs) + const sitemapRes = await fetchTextSafe(origin + '/sitemap.xml') + if (sitemapRes.ok) { + sitemapSeeds = await collectSitemapUrls(sitemapRes.text, origin, deps, maxPages, delay, delayMs) if (sitemapSeeds.length > 0) discovery = 'sitemap' } @@ -386,7 +433,7 @@ export async function crawlSite(options: CrawlOptions, deps: CrawlDeps): Promise if (html && depth < maxDepth) { for (const link of extractLinks(html, url)) { const n = normalizeUrl(link) - if (sameOrigin(n, origin) && !visited.has(n)) { + if (sameSite(n, origin) && !visited.has(n)) { queue.push({ url: n, depth: depth + 1 }) } } @@ -410,6 +457,7 @@ export async function crawlSite(options: CrawlOptions, deps: CrawlDeps): Promise startUrl: options.startUrl, pages, skipped, + skippedReasons, document, generatedAt, stats: { @@ -422,7 +470,7 @@ export async function crawlSite(options: CrawlOptions, deps: CrawlDeps): Promise } // Agrège les URLs d'un sitemap ; suit les sous-sitemaps d'un index (borné), filtre -// same-origin, normalise et déduplique, plafonné à maxPages. +// same-site (apex↔www inclus), normalise et déduplique, plafonné à maxPages. async function collectSitemapUrls( xml: string, origin: string, @@ -434,24 +482,24 @@ async function collectSitemapUrls( const parsed = parseSitemap(xml) const out: string[] = [] - const pushSameOrigin = (urls: string[]): void => { + const pushSameSite = (urls: string[]): void => { for (const u of urls) { - if (sameOrigin(u, origin)) out.push(normalizeUrl(u)) + if (sameSite(u, origin)) out.push(normalizeUrl(u)) if (out.length >= maxPages) break } } if (!parsed.isIndex) { - pushSameOrigin(parsed.urls) + pushSameSite(parsed.urls) return dedupePreserveOrder(out).slice(0, maxPages) } // Sitemap index : on récupère les sous-sitemaps (bornés) et on fusionne leurs URLs. - const subSitemaps = parsed.urls.filter((u) => sameOrigin(u, origin)).slice(0, MAX_SUBSITEMAPS) + const subSitemaps = parsed.urls.filter((u) => sameSite(u, origin)).slice(0, MAX_SUBSITEMAPS) for (const sub of subSitemaps) { if (out.length >= maxPages) break - const subXml = await deps.fetchText(sub).catch(() => null) - if (subXml) pushSameOrigin(parseSitemap(subXml).urls) + const subRes = await deps.fetchText(sub).catch((): FetchTextResult => ({ ok: false, reason: 'network' })) + if (subRes.ok) pushSameSite(parseSitemap(subRes.text).urls) await delay(delayMs) } return dedupePreserveOrder(out).slice(0, maxPages) diff --git a/src/sidepanel/store.ts b/src/sidepanel/store.ts index b404f64..0c41a30 100644 --- a/src/sidepanel/store.ts +++ b/src/sidepanel/store.ts @@ -45,6 +45,10 @@ interface PanelState { // Chosen crawl export shape. Transient (never persisted): a fresh panel always // defaults to the single-file .md; the ⌘K "Crawl → ZIP" command presets 'zip'. crawlExportMode: CrawlExportMode + // Whether the next crawl respects robots.txt. Transient (never persisted): a + // fresh panel always defaults to ON (safe default). The user can turn it OFF + // from the crawl view when robots.txt is what's blocking every page. + crawlRespectRobots: boolean // True once `persist` has finished its async rehydration from // chrome.storage.local. The panel UI gates on this so it never paints the // default state first and then snaps to the restored one (see App.tsx). @@ -68,6 +72,7 @@ interface PanelState { setCrawlResult: (result: CrawlResult | null) => void setCrawlError: (error: string | null) => void setCrawlExportMode: (mode: CrawlExportMode) => void + setCrawlRespectRobots: (respect: boolean) => void setHasHydrated: (hydrated: boolean) => void } @@ -89,6 +94,7 @@ export const usePanelStore = create()( crawlResult: null, crawlError: null, crawlExportMode: 'single', + crawlRespectRobots: true, hasHydrated: false, setMode: (mode) => set({ activeMode: mode }), @@ -109,6 +115,7 @@ export const usePanelStore = create()( setCrawlResult: (result) => set({ crawlResult: result }), setCrawlError: (error) => set({ crawlError: error }), setCrawlExportMode: (mode) => set({ crawlExportMode: mode }), + setCrawlRespectRobots: (respect) => set({ crawlRespectRobots: respect }), setHasHydrated: (hydrated) => set({ hasHydrated: hydrated }), }), { diff --git a/src/sidepanel/views/CrawlView.tsx b/src/sidepanel/views/CrawlView.tsx index ed3c0ec..e4c3b6e 100644 --- a/src/sidepanel/views/CrawlView.tsx +++ b/src/sidepanel/views/CrawlView.tsx @@ -30,6 +30,28 @@ function formatBytes(bytes: number): string { return `${(bytes / (1024 * 1024)).toFixed(1)} MB` } +// Libellé lisible d'une raison de skip pour le breakdown (`http-403` → `HTTP 403`). +// C'est ce qui dit à l'utilisateur POURQUOI une page a été ignorée. +function formatSkipReason(reason: string): string { + if (reason.startsWith('http-')) return `HTTP ${reason.slice(5)}` + switch (reason) { + case 'robots': + return 'robots.txt' + case 'timeout': + return 'timeout' + case 'non-text': + return 'non-text' + case 'cross-origin': + return 'off-site' + case 'network': + return 'network/CORS' + case 'empty': + return 'empty' + default: + return reason + } +} + function CrawlView() { const running = usePanelStore((s) => s.crawlRunning) const progress = usePanelStore((s) => s.crawlProgress) @@ -41,6 +63,8 @@ function CrawlView() { const setError = usePanelStore((s) => s.setCrawlError) const exportMode = usePanelStore((s) => s.crawlExportMode) const setExportMode = usePanelStore((s) => s.setCrawlExportMode) + const respectRobots = usePanelStore((s) => s.crawlRespectRobots) + const setRespectRobots = usePanelStore((s) => s.setCrawlRespectRobots) const [copied, setCopied] = useState(false) const handleStart = useCallback(async () => { @@ -51,7 +75,7 @@ function CrawlView() { try { // startUrl vide : le content script fait autorité et utilise location.href de // l'onglet courant (le panel n'a pas forcément accès à tab.url). - const res = await sendMessage(MessageType.CRAWL_SITE, { startUrl: '' }) + const res = await sendMessage(MessageType.CRAWL_SITE, { startUrl: '', respectRobots }) if (!res?.success) { setRunning(false) setError('unsupported') @@ -61,7 +85,7 @@ function CrawlView() { setRunning(false) setError('failed') } - }, [setRunning, setError, setResult, setProgress]) + }, [setRunning, setError, setResult, setProgress, respectRobots]) const handleStop = useCallback(() => { // Annulation coopérative : le crawl finit la page en cours puis renvoie le document @@ -175,6 +199,13 @@ function CrawlView() { // --- Done — document concaténé ---------------------------------------------------- const { stats } = result + // Breakdown lisible des raisons de skip (trié par nombre décroissant). RÉVÈLE + // pourquoi le crawl skippe — la donnée manquante de la version précédente. + const reasonEntries = (Object.entries(result.skippedReasons) as [string, number | undefined][]) + .map(([reason, n]) => [reason, n ?? 0] as const) + .filter(([, n]) => n > 0) + .sort((a, b) => b[1] - a[1]) + const robotsSkipped = result.skippedReasons.robots ?? 0 const preview = result.document.length > PREVIEW_LIMIT ? result.document.slice(0, PREVIEW_LIMIT) + @@ -248,10 +279,24 @@ function CrawlView() { })} - {stats.pageCount === 0 && ( + {reasonEntries.length > 0 && (

- No pages could be converted — the site may be client-rendered or blocked by - robots.txt. + {stats.skippedCount.toLocaleString()} skipped:{' '} + {reasonEntries.map(([reason, n]) => `${n} ${formatSkipReason(reason)}`).join(' · ')} +

+ )} + + {respectRobots && robotsSkipped > 0 && ( +

+ {robotsSkipped.toLocaleString()} blocked by robots.txt — turn off “Respect + robots.txt” below and crawl again to include them. +

+ )} + + {stats.pageCount === 0 && stats.skippedCount === 0 && ( +

+ No pages could be converted — the site may be client-rendered (its initial HTML + is empty).

)} @@ -263,6 +308,29 @@ function CrawlView() { {/* Actions */}
+ {/* Respect robots.txt — ON by default (safe). Turn OFF to include pages that + robots.txt blocks: this is a human converting pages they can already see, + not an indexing crawler. Takes effect on the next crawl (Crawl again). */} +
+ Respect robots.txt + +
+ {/* Export format — single concatenated .md vs a multi-file .zip. Additive: Copy always yields the single document; only the download shape changes. */}
`: réponse non-OK (ex. `http-403` anti-bot, `http-404`) + * - `timeout` : requête au-delà du timeout dur (serveur muet) + * - `non-text` : content-type binaire (image, pdf…) — non converti + * - `cross-origin` : hors du site (autre domaine que la variante apex/www) + * - `network` : fetch rejeté (erreur réseau, CORS, Stop en vol) + * - `empty` : converti mais Markdown vide (page sans contenu exploitable) + */ +export type SkipReason = + | 'robots' + | `http-${number}` + | 'timeout' + | 'non-text' + | 'cross-origin' + | 'network' + | 'empty' + +/** Décompte des pages ignorées par raison (raisons absentes = 0). */ +export type SkipReasonCounts = Partial> + +/** + * Résultat discriminé d'un fetch de crawl : texte lu, ou raison d'échec explicite. + * Remplace l'ancien `string | null` qui écrasait toutes les causes (403/timeout/ + * binaire/CORS) en un seul `null` opaque — ce qui rendait tout skip invisible. + */ +export type FetchTextResult = + | { ok: true; text: string } + | { ok: false; reason: SkipReason } + /** Progression émise pendant le crawl (relayée content → background → panel). */ export interface CrawlProgress { /** Pages effectivement converties jusqu'ici. */ @@ -24,8 +63,10 @@ export interface CrawlProgress { total: number /** URL en cours de traitement. */ currentUrl: string - /** Pages ignorées (robots, erreur réseau, non-HTML). */ + /** Pages ignorées (robots, erreur réseau, non-HTML…). */ skipped: number + /** Dernière raison de skip observée (feedback live ; absent si aucun skip encore). */ + lastSkipReason?: SkipReason } /** Markdown d'une page convertie (corps seul, sans frontmatter par page). */ @@ -57,6 +98,12 @@ export interface CrawlResult { pages: CrawlPageResult[] /** URLs ignorées (robots / erreur / non-HTML). */ skipped: string[] + /** + * Décompte des pages ignorées PAR RAISON — c'est CE qui dit à l'utilisateur + * pourquoi le crawl skippe (ex. `{ robots: 45, 'http-403': 3 }`). Toujours présent + * (peut être vide `{}` si aucun skip). + */ + skippedReasons: SkipReasonCounts /** Le gros .md concaténé : frontmatter global + table des matières + sections par page. */ document: string /** ISO 8601 — instant de fin de crawl. */ From 0e604a9fe69521470e00104992bc5fc26b3ce67e Mon Sep 17 00:00:00 2001 From: Adam Date: Wed, 1 Jul 2026 04:44:15 +0200 Subject: [PATCH 5/5] feat(crawl): Full real-browser render for JS sites + fix stale cross-site display - Crawl 'Full' mode renders each page in a background tab so the site's JS runs, captures the rendered DOM via the content script's EXTRACT_MARKDOWN, then closes the tab. Fixes SPA sites that returned empty pages in fetch mode. Sequential, guaranteed tab cleanup, 20s per-page timeout, never steals focus. Fast/Full toggle + 'empty -> try Full' hint. Adds the 'tabs' permission. - Fix stale scan: the panel ties the shown scan/markdown to the active tab's URL; a non-destructive banner appears when showing a scan from another host (updates on tab switch thanks to the new tabs permission). - Unit tests 218 -> 245. --- manifest.json | 1 + src/background/service-worker.ts | 94 +++++++++++ src/content/index.ts | 40 ++++- src/lib/__tests__/crawler.test.ts | 138 ++++++++++++++- src/lib/__tests__/tab-render-sw.test.ts | 169 +++++++++++++++++++ src/lib/__tests__/tab-render.test.ts | 137 +++++++++++++++ src/lib/__tests__/url.test.ts | 34 ++++ src/lib/crawler.ts | 59 +++++-- src/lib/tab-render.ts | 100 +++++++++++ src/lib/url.ts | 36 ++++ src/sidepanel/App.tsx | 58 +++++++ src/sidepanel/components/StaleScanBanner.tsx | 44 +++++ src/sidepanel/store.ts | 9 + src/sidepanel/views/CrawlView.tsx | 87 +++++++++- src/sidepanel/views/DesignSystemView.tsx | 10 ++ src/sidepanel/views/MarkdownView.tsx | 10 ++ src/sidepanel/views/ScanView.tsx | 11 ++ src/types/crawl.ts | 21 +++ src/types/messages.ts | 14 +- 19 files changed, 1052 insertions(+), 20 deletions(-) create mode 100644 src/lib/__tests__/tab-render-sw.test.ts create mode 100644 src/lib/__tests__/tab-render.test.ts create mode 100644 src/lib/__tests__/url.test.ts create mode 100644 src/lib/tab-render.ts create mode 100644 src/lib/url.ts create mode 100644 src/sidepanel/components/StaleScanBanner.tsx diff --git a/manifest.json b/manifest.json index 9a140b7..2b6e215 100644 --- a/manifest.json +++ b/manifest.json @@ -12,6 +12,7 @@ "permissions": [ "activeTab", "scripting", + "tabs", "sidePanel", "storage", "clipboardWrite" diff --git a/src/background/service-worker.ts b/src/background/service-worker.ts index fb717a8..bc92fb2 100644 --- a/src/background/service-worker.ts +++ b/src/background/service-worker.ts @@ -2,7 +2,9 @@ import { MessageType } from '@/types/messages' import type { MessagePayloadMap } from '@/types/messages' +import type { MarkdownResult } from '@/types/markdown' import { saveDesignSystem } from '@/lib/storage' +import { renderPageInTab, type TabRenderDeps } from '@/lib/tab-render' // Prevent side panel from opening on action click (we control it manually) chrome.sidePanel.setPanelBehavior({ openPanelOnActionClick: false }) @@ -145,6 +147,16 @@ chrome.runtime.onMessage.addListener((message: IncomingMessage, sender, sendResp return true // async response } + case MessageType.RENDER_PAGE: { + // Full-render mode: the content script (no chrome.tabs) delegates ONE page here. + // We open it in a background tab, let the JS run, capture the RENDERED DOM via + // EXTRACT_MARKDOWN, collect its links, then close the tab. renderPageInTab never + // rejects (returns a discriminated reason) and always closes the tab (no orphan). + const { url } = payload as MessagePayloadMap[MessageType.RENDER_PAGE] + renderPageInTab(url, realTabRenderDeps).then((outcome) => sendResponse(outcome)) + return true // async response + } + // CRAWL_PROGRESS / CRAWL_COMPLETE are intentionally NOT handled here: the content // script broadcasts them and the side panel listens directly. Relaying them // through the worker would double-deliver the (potentially multi-MB) document. @@ -244,3 +256,85 @@ export async function ensureContentScriptAndForward( if (!tab?.id) return false return ensureContentScriptOnTab(tab.id, type, payload) } + +// --- Full-render tab orchestration (RENDER_PAGE) ------------------------------------ +// +// Real chrome.* wiring for lib/tab-render. Kept here (not in the message handler) so the +// orchestration in lib/tab-render stays pure/testable and this file just injects the APIs. + +// Resolves when `tabId` reaches status 'complete'. Also resolves if the tab is REMOVED +// (e.g. the per-page timeout in renderPageInTab already closed it): this lets the +// abandoned render step settle and, crucially, removes both listeners so nothing leaks. +// Guards against the load event having already fired before we subscribed (checks the +// current status once via tabs.get). +export function waitForTabComplete(tabId: number): Promise { + return new Promise((resolve) => { + let settled = false + const finish = (): void => { + if (settled) return + settled = true + chrome.tabs.onUpdated.removeListener(onUpdated) + chrome.tabs.onRemoved.removeListener(onRemoved) + resolve() + } + const onUpdated = (id: number, info: chrome.tabs.OnUpdatedInfo): void => { + if (id === tabId && info.status === 'complete') finish() + } + const onRemoved = (id: number): void => { + if (id === tabId) finish() + } + chrome.tabs.onUpdated.addListener(onUpdated) + chrome.tabs.onRemoved.addListener(onRemoved) + // If the tab was already 'complete' before we subscribed, the event won't fire again. + chrome.tabs + .get(tabId) + .then((tab) => { + if (tab.status === 'complete') finish() + }) + .catch(() => finish()) + }) +} + +// Real dependencies handed to renderPageInTab. active:false = never steals focus. +export const realTabRenderDeps: TabRenderDeps = { + createTab: async (url: string): Promise => { + const tab = await chrome.tabs.create({ url, active: false }) + if (tab.id === undefined) throw new Error('tab created without an id') + return tab.id + }, + waitForComplete: waitForTabComplete, + extractMarkdown: async (tabId: number): Promise => { + // The content script auto-injects on load; re-inject defensively if MV3 + // never ran it on this fresh tab, then ask it to convert the RENDERED DOM. + if (!(await pingContentScript(tabId))) { + if (!(await injectContentScript(tabId))) return undefined + } + try { + return (await chrome.tabs.sendMessage(tabId, { + type: MessageType.EXTRACT_MARKDOWN, + payload: undefined, + })) as MarkdownResult | undefined + } catch (err) { + console.debug('[PixelLens]', (err as Error).message) + return undefined + } + }, + collectLinks: async (tabId: number): Promise => { + // Absolute hrefs from the RENDERED DOM — feeds the BFS when the site has no sitemap. + try { + const [res] = await chrome.scripting.executeScript({ + target: { tabId }, + func: () => + Array.from(document.querySelectorAll('a[href]')) + .map((a) => (a as HTMLAnchorElement).href) + .filter((h) => h.startsWith('http')), + }) + return (res?.result as string[] | undefined) ?? [] + } catch (err) { + console.debug('[PixelLens]', (err as Error).message) + return [] + } + }, + removeTab: (tabId: number): Promise => chrome.tabs.remove(tabId).then(() => {}), +} + diff --git a/src/content/index.ts b/src/content/index.ts index 40adf8f..7b08418 100644 --- a/src/content/index.ts +++ b/src/content/index.ts @@ -2,6 +2,8 @@ import { onMessage } from '@/lib/messaging' import { MessageType } from '@/types/messages' +import type { CrawlDeps } from '@/lib/crawler' +import type { RenderPageResult } from '@/types/crawl' import { ElementHighlighter } from './inspector/ElementHighlighter' import { ElementSelector } from './inspector/ElementSelector' import { DistanceMeasurer } from './inspector/DistanceMeasurer' @@ -173,11 +175,13 @@ onMessage(MessageType.CRAWL_SITE, (options, _sender, sendResponse) => { // Le content script fait autorité sur l'URL de départ : si le panel n'a pas pu // fournir tab.url, on prend location.href de la page courante. const startUrl = options.startUrl || location.href - const result = await crawlSite({ ...options, startUrl }, { + + const deps: CrawlDeps = { // Chaque requête a un timeout dur ET écoute le signal d'annulation global : // Stop (STOP_CRAWL) abort() le fetch en vol, un timeout coupe un serveur muet. // Une page en échec (timeout/abort/erreur) renvoie null → comptée « skipped », - // le crawl continue. + // le crawl continue. En mode Full, fetchText ne sert QUE à la découverte + // (robots.txt / sitemap.xml, statiques et rendu-indépendants). fetchText: (url) => fetchTextWithTimeout(url, { signal: crawlStopController?.signal }), convert: (html, url) => { @@ -200,7 +204,37 @@ onMessage(MessageType.CRAWL_SITE, (options, _sender, sendResponse) => { sendMessage(MessageType.CRAWL_PROGRESS, progress).catch(() => {}) }, shouldStop: () => crawlAborted, - }) + } + + // Mode Full : le CONTENU de chaque page vient d'une navigation réelle. Le content + // script ne peut pas créer d'onglets → on délègue au service worker (RENDER_PAGE), + // qui rend la page dans un onglet arrière-plan et renvoie le Markdown du DOM rendu. + if (options.renderMode === 'full') { + deps.renderPage = async (url): Promise => { + try { + const res = await sendMessage(MessageType.RENDER_PAGE, { url }) + if (!res || !res.ok) { + return { ok: false, reason: res ? res.reason : 'network' } + } + const body = res.markdown.markdown.trim() + if (!body) return { ok: false, reason: 'empty' } + return { + ok: true, + page: { + url, + title: res.markdown.frontmatter.title, + markdown: body, + wordCount: res.markdown.frontmatter.wordCount, + }, + links: res.links, + } + } catch { + return { ok: false, reason: 'network' } + } + } + } + + const result = await crawlSite({ ...options, startUrl }, deps) sendMessage(MessageType.CRAWL_COMPLETE, { result }).catch(() => {}) }) .catch((err) => console.debug('[PixelLens]', (err as Error).message)) diff --git a/src/lib/__tests__/crawler.test.ts b/src/lib/__tests__/crawler.test.ts index ea2f729..e60b3e0 100644 --- a/src/lib/__tests__/crawler.test.ts +++ b/src/lib/__tests__/crawler.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect } from 'vitest' +import { describe, it, expect, vi } from 'vitest' import { normalizeUrl, sameOrigin, @@ -12,7 +12,7 @@ import { type CrawlDeps, } from '../crawler' import { htmlDocumentToMarkdown } from '../markdown' -import type { CrawlPageResult, FetchTextResult } from '@/types/crawl' +import type { CrawlPageResult, FetchTextResult, RenderPageResult, SkipReason } from '@/types/crawl' // --- Helpers ------------------------------------------------------------------------ @@ -519,6 +519,140 @@ describe('crawlSite — breakdown des raisons de skip', () => { }) }) +// --- crawlSite : mode Full (renderPage — rendu par onglet réel) --------------------- + +// Fake renderPage piloté par une table URL -> { md?, links?, reason? } : présent+md -> +// succès (page + liens du DOM rendu), reason -> échec discriminé, absent -> network. +function makeRenderPage( + table: Record, +) { + const calls: string[] = [] + const renderPage = async (url: string): Promise => { + calls.push(url) + const entry = table[url] + if (!entry || entry.reason) return { ok: false, reason: entry?.reason ?? 'network' } + return { + ok: true, + page: { url, title: `Rendered ${url}`, markdown: entry.md ?? `Body ${url}`, wordCount: 2 }, + links: entry.links ?? [], + } + } + return { renderPage, calls } +} + +describe('crawlSite — mode Full (renderPage)', () => { + it('prend le CONTENU des pages via renderPage, jamais via fetch/convert', async () => { + const fetchCalls: string[] = [] + const fetchText = async (url: string): Promise => { + fetchCalls.push(url) + if (url === 'https://spa.example/sitemap.xml') { + return { + ok: true, + text: + 'https://spa.example/a' + + 'https://spa.example/b', + } + } + return { ok: false, reason: 'network' } // robots.txt absent + } + // convert (le chemin Fast HTML→Markdown) renverrait « empty » sur une SPA : il ne doit + // JAMAIS être appelé en mode Full. + const convert = vi.fn(() => null) + const { renderPage, calls } = makeRenderPage({ + 'https://spa.example/': { md: 'Home rendered' }, + 'https://spa.example/a': { md: 'A rendered' }, + 'https://spa.example/b': { md: 'B rendered' }, + }) + + const res = await crawlSite( + { startUrl: 'https://spa.example/', renderMode: 'full' }, + { fetchText, convert, renderPage, delay: () => Promise.resolve() }, + ) + + expect(res.stats.discovery).toBe('sitemap') + expect(sortedPageUrls(res.pages)).toEqual([ + 'https://spa.example/', + 'https://spa.example/a', + 'https://spa.example/b', + ]) + // fetch ne touche QUE la découverte (robots + sitemap), pas les pages elles-mêmes. + expect(fetchCalls).toEqual([ + 'https://spa.example/robots.txt', + 'https://spa.example/sitemap.xml', + ]) + expect(convert).not.toHaveBeenCalled() + expect(calls).toContain('https://spa.example/a') + }) + + it('produit du Markdown NON vide là où le Fast donnait « empty » (cas SPA)', async () => { + const fetchText = async (url: string): Promise => + url === 'https://spa.example/sitemap.xml' + ? { ok: true, text: 'https://spa.example/a' } + : { ok: false, reason: 'network' } + const { renderPage } = makeRenderPage({ + 'https://spa.example/': { md: '# Real home content' }, + 'https://spa.example/a': { md: '# Real A content' }, + }) + + const res = await crawlSite( + { startUrl: 'https://spa.example/', renderMode: 'full' }, + // convert renverrait null (shell vide) en Fast : ici on prouve que Full l'ignore. + { fetchText, convert: () => null, renderPage, delay: () => Promise.resolve() }, + ) + + expect(res.pages).toHaveLength(2) + expect(res.pages.every((p) => p.markdown.trim().length > 0)).toBe(true) + expect(res.skippedReasons.empty ?? 0).toBe(0) + }) + + it('skippe une page dont le rendu échoue, avec sa raison (timeout)', async () => { + const fetchText = async (url: string): Promise => + url === 'https://spa.example/sitemap.xml' + ? { + ok: true, + text: + 'https://spa.example/slow' + + 'https://spa.example/ok', + } + : { ok: false, reason: 'network' } + const { renderPage } = makeRenderPage({ + 'https://spa.example/': { md: 'Home' }, + 'https://spa.example/slow': { reason: 'timeout' }, + 'https://spa.example/ok': { md: 'OK' }, + }) + + const res = await crawlSite( + { startUrl: 'https://spa.example/', renderMode: 'full' }, + { fetchText, convert: () => null, renderPage, delay: () => Promise.resolve() }, + ) + + expect(res.skippedReasons.timeout).toBe(1) + expect(sortedPageUrls(res.pages)).toEqual(['https://spa.example/', 'https://spa.example/ok']) + }) + + it('sans sitemap, étend le BFS via les liens du DOM RENDU', async () => { + // Ni robots ni sitemap : découverte par BFS. Les liens viennent du DOM rendu. + const fetchText = async (): Promise => ({ ok: false, reason: 'network' }) + const { renderPage, calls } = makeRenderPage({ + 'https://spa.example/': { md: 'Home', links: ['https://spa.example/deep'] }, + 'https://spa.example/deep': { md: 'Deep', links: [] }, + }) + + const res = await crawlSite( + { startUrl: 'https://spa.example/', renderMode: 'full' }, + { fetchText, convert: () => null, renderPage, delay: () => Promise.resolve() }, + ) + + expect(res.stats.discovery).toBe('crawl') + expect(sortedPageUrls(res.pages)).toEqual([ + 'https://spa.example/', + 'https://spa.example/deep', + ]) + expect(calls).toContain('https://spa.example/deep') + }) +}) + + // --- Réutilisation du moteur Markdown sur un Document fetché ------------------------- describe('htmlDocumentToMarkdown (page fetchée via DOMParser)', () => { diff --git a/src/lib/__tests__/tab-render-sw.test.ts b/src/lib/__tests__/tab-render-sw.test.ts new file mode 100644 index 0000000..ffed435 --- /dev/null +++ b/src/lib/__tests__/tab-render-sw.test.ts @@ -0,0 +1,169 @@ +import { describe, it, expect, beforeEach, vi } from 'vitest' +import type { MarkdownResult } from '@/types/markdown' + +// The service worker runs chrome.* calls at module-eval time. Wire a chrome mock via +// vi.hoisted so it exists before the SW module is imported. onUpdated/onRemoved capture +// their listeners so tests can fire the events by hand. +const mocks = vi.hoisted(() => { + const updatedListeners: ((id: number, info: { status?: string }) => void)[] = [] + const removedListeners: ((id: number) => void)[] = [] + const tabsCreate = vi.fn() + const tabsRemove = vi.fn() + const tabsGet = vi.fn() + const tabsSendMessage = vi.fn() + const executeScript = vi.fn() + ;(globalThis as unknown as { chrome: unknown }).chrome = { + sidePanel: { setPanelBehavior: vi.fn() }, + commands: { onCommand: { addListener: vi.fn() } }, + runtime: { + onMessage: { addListener: vi.fn() }, + sendMessage: vi.fn(), + getManifest: () => ({ content_scripts: [{ js: ['content.js'] }] }), + }, + tabs: { + create: tabsCreate, + remove: tabsRemove, + get: tabsGet, + sendMessage: tabsSendMessage, + query: vi.fn(), + onUpdated: { + addListener: (f: (id: number, info: { status?: string }) => void) => updatedListeners.push(f), + removeListener: (f: (id: number, info: { status?: string }) => void) => { + const i = updatedListeners.indexOf(f) + if (i >= 0) updatedListeners.splice(i, 1) + }, + }, + onRemoved: { + addListener: (f: (id: number) => void) => removedListeners.push(f), + removeListener: (f: (id: number) => void) => { + const i = removedListeners.indexOf(f) + if (i >= 0) removedListeners.splice(i, 1) + }, + }, + }, + scripting: { executeScript }, + storage: { sync: { get: vi.fn(), set: vi.fn() } }, + action: { setBadgeText: vi.fn(), setBadgeBackgroundColor: vi.fn() }, + } + return { + updatedListeners, + removedListeners, + tabsCreate, + tabsRemove, + tabsGet, + tabsSendMessage, + executeScript, + } +}) + +import { realTabRenderDeps, waitForTabComplete } from '@/background/service-worker' +import { MessageType } from '@/types/messages' + +const md: MarkdownResult = { + frontmatter: { title: 'T', url: 'https://x.test/', capturedAt: 'now', wordCount: 1 }, + markdown: '# T', + fullDocument: '---\n---\n\n# T', + stats: { headings: 1, links: 0, images: 0, tables: 0, codeBlocks: 0 }, +} + +beforeEach(() => { + mocks.tabsCreate.mockReset() + mocks.tabsRemove.mockReset() + mocks.tabsGet.mockReset() + mocks.tabsSendMessage.mockReset() + mocks.executeScript.mockReset() + mocks.updatedListeners.length = 0 + mocks.removedListeners.length = 0 + vi.spyOn(console, 'debug').mockImplementation(() => {}) +}) + +describe('realTabRenderDeps.createTab', () => { + it('opens a BACKGROUND tab (active:false) and returns its id', async () => { + mocks.tabsCreate.mockResolvedValue({ id: 77 }) + const id = await realTabRenderDeps.createTab('https://x.test/') + expect(id).toBe(77) + // active:false is the "never steals focus" guarantee. + expect(mocks.tabsCreate).toHaveBeenCalledWith({ url: 'https://x.test/', active: false }) + }) + + it('throws when the created tab has no id (so renderPageInTab reports network)', async () => { + mocks.tabsCreate.mockResolvedValue({}) + await expect(realTabRenderDeps.createTab('https://x.test/')).rejects.toThrow() + }) +}) + +describe('waitForTabComplete', () => { + it('resolves when onUpdated fires status=complete for the tab, then removes its listeners', async () => { + mocks.tabsGet.mockResolvedValue({ status: 'loading' }) + const p = waitForTabComplete(5) + mocks.updatedListeners.forEach((f) => f(5, { status: 'complete' })) + await expect(p).resolves.toBeUndefined() + expect(mocks.updatedListeners).toHaveLength(0) + expect(mocks.removedListeners).toHaveLength(0) + }) + + it('ignores complete events for OTHER tabs', async () => { + mocks.tabsGet.mockResolvedValue({ status: 'loading' }) + const p = waitForTabComplete(5) + mocks.updatedListeners.forEach((f) => f(999, { status: 'complete' })) + // Still pending → now fire the right tab. + mocks.updatedListeners.forEach((f) => f(5, { status: 'complete' })) + await expect(p).resolves.toBeUndefined() + }) + + it('resolves immediately when the tab is already complete at subscription time', async () => { + mocks.tabsGet.mockResolvedValue({ status: 'complete' }) + await expect(waitForTabComplete(9)).resolves.toBeUndefined() + }) + + it('resolves when the tab is removed (lets an abandoned render settle)', async () => { + mocks.tabsGet.mockResolvedValue({ status: 'loading' }) + const p = waitForTabComplete(3) + mocks.removedListeners.forEach((f) => f(3)) + await expect(p).resolves.toBeUndefined() + }) +}) + +describe('realTabRenderDeps.extractMarkdown', () => { + it('pings then requests EXTRACT_MARKDOWN and returns the rendered-DOM result', async () => { + mocks.tabsSendMessage.mockImplementation((_id: number, msg: { type: MessageType }) => + msg.type === MessageType.PING ? Promise.resolve({ alive: true }) : Promise.resolve(md), + ) + const out = await realTabRenderDeps.extractMarkdown(11) + expect(out).toEqual(md) + expect(mocks.tabsSendMessage).toHaveBeenCalledWith(11, { + type: MessageType.EXTRACT_MARKDOWN, + payload: undefined, + }) + }) + + it('returns undefined when the content script is unreachable and cannot be injected', async () => { + mocks.tabsSendMessage.mockRejectedValue(new Error('Could not establish connection')) + mocks.executeScript.mockRejectedValue(new Error('Cannot access a chrome:// URL')) + expect(await realTabRenderDeps.extractMarkdown(12)).toBeUndefined() + }) +}) + +describe('realTabRenderDeps.collectLinks', () => { + it('returns the hrefs collected from the rendered DOM', async () => { + mocks.executeScript.mockResolvedValue([{ result: ['https://x.test/a', 'https://x.test/b'] }]) + const links = await realTabRenderDeps.collectLinks(4) + expect(links).toEqual(['https://x.test/a', 'https://x.test/b']) + expect(mocks.executeScript).toHaveBeenCalledWith( + expect.objectContaining({ target: { tabId: 4 } }), + ) + }) + + it('degrades to [] when scripting is blocked', async () => { + mocks.executeScript.mockRejectedValue(new Error('blocked')) + expect(await realTabRenderDeps.collectLinks(4)).toEqual([]) + }) +}) + +describe('realTabRenderDeps.removeTab', () => { + it('closes the tab', async () => { + mocks.tabsRemove.mockResolvedValue(undefined) + await realTabRenderDeps.removeTab(8) + expect(mocks.tabsRemove).toHaveBeenCalledWith(8) + }) +}) diff --git a/src/lib/__tests__/tab-render.test.ts b/src/lib/__tests__/tab-render.test.ts new file mode 100644 index 0000000..a61a1f2 --- /dev/null +++ b/src/lib/__tests__/tab-render.test.ts @@ -0,0 +1,137 @@ +import { describe, it, expect, vi } from 'vitest' +import { renderPageInTab, type TabRenderDeps } from '../tab-render' +import type { MarkdownResult } from '@/types/markdown' + +// --- Helpers ------------------------------------------------------------------------ + +const md = (title = 'Home', body = '# Home\n\nHello'): MarkdownResult => ({ + frontmatter: { title, url: 'https://spa.example/', capturedAt: 'now', wordCount: 2 }, + markdown: body, + fullDocument: `---\n---\n\n${body}`, + stats: { headings: 1, links: 0, images: 0, tables: 0, codeBlocks: 0 }, +}) + +// Records the order of dep calls so we can assert create→wait→extract→links→remove. +function tracingDeps(over: Partial = {}): { + deps: TabRenderDeps + calls: string[] + removed: number[] +} { + const calls: string[] = [] + const removed: number[] = [] + const deps: TabRenderDeps = { + createTab: async () => { + calls.push('createTab') + return 42 + }, + waitForComplete: async () => { + calls.push('waitForComplete') + }, + extractMarkdown: async () => { + calls.push('extractMarkdown') + return md() + }, + collectLinks: async () => { + calls.push('collectLinks') + return ['https://spa.example/about'] + }, + removeTab: async (id) => { + calls.push('removeTab') + removed.push(id) + }, + // Instant render delay so tests never wait on the real 1200ms. + delay: () => Promise.resolve(), + ...over, + } + return { deps, calls, removed } +} + +// --- renderPageInTab ---------------------------------------------------------------- + +describe('renderPageInTab — happy path', () => { + it('opens a tab, renders, extracts the rendered DOM, collects links, closes it', async () => { + const { deps, removed } = tracingDeps() + const out = await renderPageInTab('https://spa.example/', deps) + + expect(out).toEqual({ + ok: true, + markdown: md(), + links: ['https://spa.example/about'], + }) + // Tab is always closed with the id returned by createTab (no orphan). + expect(removed).toEqual([42]) + }) + + it('runs the steps in order: create → wait → extract → links → remove', async () => { + const { deps, calls } = tracingDeps() + await renderPageInTab('https://spa.example/', deps) + expect(calls).toEqual([ + 'createTab', + 'waitForComplete', + 'extractMarkdown', + 'collectLinks', + 'removeTab', + ]) + }) + + it('passes the render delay through before extracting', async () => { + const delay = vi.fn().mockResolvedValue(undefined) + const { deps } = tracingDeps({ delay, renderDelayMs: 900 }) + await renderPageInTab('https://spa.example/', deps) + expect(delay).toHaveBeenCalledWith(900) + }) +}) + +describe('renderPageInTab — skips & failures', () => { + it('returns `empty` when the rendered DOM yields no markdown, and still closes the tab', async () => { + const { deps, removed } = tracingDeps({ extractMarkdown: async () => undefined }) + const out = await renderPageInTab('https://spa.example/', deps) + expect(out).toEqual({ ok: false, reason: 'empty' }) + expect(removed).toEqual([42]) + }) + + it('returns `network` when a render step throws, and still closes the tab', async () => { + const { deps, removed } = tracingDeps({ + waitForComplete: async () => { + throw new Error('navigation failed') + }, + }) + const out = await renderPageInTab('https://spa.example/', deps) + expect(out).toEqual({ ok: false, reason: 'network' }) + expect(removed).toEqual([42]) // cleanup guaranteed on error + }) + + it('returns `timeout` when the page never finishes loading, and still closes the tab', async () => { + // waitForComplete never resolves → the hard per-page timeout must win and close the tab. + const { deps, removed } = tracingDeps({ + waitForComplete: () => new Promise(() => {}), + timeoutMs: 10, + }) + const out = await renderPageInTab('https://spa.example/', deps) + expect(out).toEqual({ ok: false, reason: 'timeout' }) + expect(removed).toEqual([42]) + }) + + it('returns `network` and closes NO tab when the tab could not be created', async () => { + const removeTab = vi.fn().mockResolvedValue(undefined) + const { deps } = tracingDeps({ + createTab: async () => { + throw new Error('cannot create tab') + }, + removeTab, + }) + const out = await renderPageInTab('https://spa.example/', deps) + expect(out).toEqual({ ok: false, reason: 'network' }) + expect(removeTab).not.toHaveBeenCalled() // no tab id → nothing to clean up + }) + + it('degrades to no links (still ok) when link collection fails', async () => { + const { deps } = tracingDeps({ + collectLinks: async () => { + throw new Error('scripting blocked') + }, + }) + const out = await renderPageInTab('https://spa.example/', deps) + expect(out).toEqual({ ok: true, markdown: md(), links: [] }) + }) +}) diff --git a/src/lib/__tests__/url.test.ts b/src/lib/__tests__/url.test.ts new file mode 100644 index 0000000..9246fae --- /dev/null +++ b/src/lib/__tests__/url.test.ts @@ -0,0 +1,34 @@ +import { describe, it, expect } from 'vitest' +import { getHost, isDifferentHost } from '@/lib/url' + +describe('getHost', () => { + it('returns the bare hostname, dropping a leading www.', () => { + expect(getHost('https://www.example.com/path?q=1')).toBe('example.com') + expect(getHost('https://example.com')).toBe('example.com') + expect(getHost('https://sub.example.com/a')).toBe('sub.example.com') + }) + + it('returns null for empty / nullish / unparseable input', () => { + expect(getHost(null)).toBeNull() + expect(getHost(undefined)).toBeNull() + expect(getHost('')).toBeNull() + expect(getHost('not a url')).toBeNull() + }) +}) + +describe('isDifferentHost', () => { + it('is true only when both hosts are known AND differ', () => { + expect(isDifferentHost('https://a.com', 'https://b.com')).toBe(true) + expect(isDifferentHost('https://www.a.com', 'https://a.com')).toBe(false) + expect(isDifferentHost('https://a.com/x', 'https://a.com/y')).toBe(false) + }) + + it('never flags a mismatch when either host is unknown', () => { + // Guards against a false "stale scan" banner when the active tab URL can't + // be read (e.g. no host permission) or the data carries no URL. + expect(isDifferentHost('https://a.com', null)).toBe(false) + expect(isDifferentHost(null, 'https://a.com')).toBe(false) + expect(isDifferentHost('https://a.com', 'chrome://newtab')).toBe(false) + expect(isDifferentHost(undefined, undefined)).toBe(false) + }) +}) diff --git a/src/lib/crawler.ts b/src/lib/crawler.ts index 464e6e8..54f9049 100644 --- a/src/lib/crawler.ts +++ b/src/lib/crawler.ts @@ -14,9 +14,13 @@ // content script ET en jsdom) : tout l'I/O (fetch) et la conversion HTML→Markdown sont // INJECTÉS via `CrawlDeps`, ce qui rend l'orchestration entièrement testable hors réseau. // -// LIMITE CONNUE (V1) : rendu fetch-first. On parse le HTML initial renvoyé par le serveur. -// Les SPA rendues 100% côté client (JS) ne livrent que leur HTML de démarrage (souvent -// vide) — leur contenu réel n'est pas capturé. Mode onglet = V2 (non implémenté). +// MODES DE RENDU : +// - Fast (défaut) : rendu fetch-first. On parse le HTML initial renvoyé par le serveur +// (deps.fetchText + deps.convert). Les SPA rendues 100% côté client ne livrent que +// leur shell de démarrage (souvent vide → skip `empty`). +// - Full : si `deps.renderPage` est fourni, le contenu de CHAQUE page vient d'une +// NAVIGATION réelle dans un onglet (DOM rendu, JS exécuté) — voir lib/tab-render et le +// service worker. La découverte (robots.txt / sitemap.xml) reste en fetch same-origin. import type { CrawlDiscovery, @@ -25,6 +29,7 @@ import type { CrawlProgress, CrawlResult, FetchTextResult, + RenderPageResult, SkipReason, SkipReasonCounts, } from '@/types/crawl' @@ -45,6 +50,12 @@ export interface CrawlDeps { fetchText: (url: string) => Promise /** Convertit le HTML d'une page en Markdown ; `null` si rien d'exploitable. */ convert: (html: string, url: string) => CrawlPageResult | null + /** + * Mode Full (optionnel) : rend UNE page dans un onglet réel et renvoie son Markdown + * (DOM RENDU) + les liens du DOM (pour le BFS). Quand fourni, il REMPLACE fetchText+ + * convert pour le CONTENU des pages ; la découverte robots/sitemap reste en fetch. + */ + renderPage?: (url: string) => Promise onProgress?: (progress: CrawlProgress) => void /** Annulation coopérative : interrogé avant chaque page. */ shouldStop?: () => boolean @@ -366,10 +377,14 @@ export async function crawlSite(options: CrawlOptions, deps: CrawlDeps): Promise lastSkipReason = reason } - // Traite une URL : garde same-site + robots, fetch, convert. Renvoie le HTML - // (pour l'expansion de liens en BFS) ou null si la page a été ignorée — avec, dans - // ce cas, une RAISON explicite enregistrée dans skippedReasons. - const processOne = async (url: string, total: number): Promise => { + // Résultat d'une page traitée avec succès : de quoi étendre le BFS. Fast = HTML serveur + // (les liens en seront extraits à la demande) ; Full = liens déjà collectés du DOM rendu. + type Processed = { html: string } | { links: string[] } + + // Traite une URL : garde same-site + robots, puis récupère son CONTENU. Renvoie de quoi + // étendre le BFS (`html` en mode Fast, `links` du DOM rendu en mode Full) ou null si la + // page a été ignorée — avec, dans ce cas, une RAISON explicite dans skippedReasons. + const processOne = async (url: string, total: number): Promise => { if (!sameSite(url, origin)) { skip(url, 'cross-origin') return null @@ -379,6 +394,26 @@ export async function crawlSite(options: CrawlOptions, deps: CrawlDeps): Promise return null } emit(url, total) + + // Mode Full : le contenu vient d'une NAVIGATION réelle (DOM rendu), pas du HTML serveur. + // Le BFS s'étend alors depuis les liens du DOM rendu (renvoyés par renderPage). + if (deps.renderPage) { + const rendered = await deps.renderPage(url) + if (!rendered.ok) { + skip(url, rendered.reason) + return null + } + if (!rendered.page.markdown.trim()) { + skip(url, 'empty') + return null + } + pages.push(rendered.page) + emit(url, total) + await delay(delayMs) + return { links: rendered.links } + } + + // Mode Fast : HTML initial du serveur (fetch same-origin) → conversion DOMParser. const res = await fetchTextSafe(url) if (!res.ok) { skip(url, res.reason) @@ -393,7 +428,7 @@ export async function crawlSite(options: CrawlOptions, deps: CrawlDeps): Promise pages.push(page) emit(url, total) await delay(delayMs) - return html + return { html } } // --- Découverte : sitemap.xml d'abord --- @@ -429,9 +464,11 @@ export async function crawlSite(options: CrawlOptions, deps: CrawlDeps): Promise if (visited.has(url)) continue visited.add(url) const total = Math.min(pages.length + queue.length + 1, maxPages) - const html = await processOne(url, total) - if (html && depth < maxDepth) { - for (const link of extractLinks(html, url)) { + const processed = await processOne(url, total) + if (processed && depth < maxDepth) { + // Fast : liens extraits du HTML serveur. Full : liens déjà collectés du DOM rendu. + const links = 'links' in processed ? processed.links : extractLinks(processed.html, url) + for (const link of links) { const n = normalizeUrl(link) if (sameSite(n, origin) && !visited.has(n)) { queue.push({ url: n, depth: depth + 1 }) diff --git a/src/lib/tab-render.ts b/src/lib/tab-render.ts new file mode 100644 index 0000000..dbc1cb3 --- /dev/null +++ b/src/lib/tab-render.ts @@ -0,0 +1,100 @@ +// PixelLens — Rendu d'une page dans un onglet réel (mode crawl "Full") +// +// POURQUOI : le mode Fast fetch le HTML initial du serveur. Sur une SPA (rendue 100% en +// JS), ce HTML est un shell quasi-vide → conversion `empty`. Le mode Full NAVIGUE +// réellement chaque page dans un onglet en arrière-plan : le navigateur exécute le JS, +// puis on capture le DOM RENDU (via EXTRACT_MARKDOWN, le même MarkdownExtractor que la +// vue Markdown). Bonus : c'est le navigateur de l'utilisateur → sa session/ses cookies +// passent les challenges JS passifs. +// +// OÙ : seul le service worker a chrome.tabs (le content script ne peut pas créer +// d'onglets). Ce module isole l'orchestration d'UNE page — tout l'I/O chrome.* est +// INJECTÉ via TabRenderDeps, ce qui rend le cycle create→wait→extract→close entièrement +// testable hors navigateur (mocks). Le service worker câble les vraies implémentations. +// +// GARANTIES : séquentiel (une page à la fois, piloté par le crawler), timeout dur global +// par page (un onglet qui ne charge/rend jamais ne fige pas le crawl), et nettoyage +// TOUJOURS effectué (finally) — aucun onglet orphelin, même sur timeout/erreur. + +import type { MarkdownResult } from '@/types/markdown' +import type { RenderPageResponse } from '@/types/messages' + +/** Délai (ms) après `complete` pour laisser le JS de la SPA finir de peindre le DOM. */ +export const DEFAULT_RENDER_DELAY_MS = 1200 +/** Timeout dur global par page (ms) : dépassé → outcome `timeout`, onglet fermé. */ +export const DEFAULT_RENDER_TIMEOUT_MS = 20_000 + +/** I/O chrome.* injecté — permet de tester l'orchestration sans navigateur. */ +export interface TabRenderDeps { + /** Ouvre un onglet EN ARRIÈRE-PLAN (active:false) sur `url` ; résout avec son id. */ + createTab: (url: string) => Promise + /** Résout quand l'onglet a fini de charger ('complete') — ou quand il est fermé. */ + waitForComplete: (tabId: number) => Promise + /** Envoie EXTRACT_MARKDOWN au content script de l'onglet ; résout le MarkdownResult. */ + extractMarkdown: (tabId: number) => Promise + /** Collecte les liens absolus (href) du DOM RENDU de l'onglet (pour le BFS Full). */ + collectLinks: (tabId: number) => Promise + /** Ferme l'onglet. TOUJOURS appelé (même sur timeout/erreur) → pas d'orphelin. */ + removeTab: (tabId: number) => Promise + /** Délai après `complete` avant capture (défaut DEFAULT_RENDER_DELAY_MS). */ + renderDelayMs?: number + /** Timeout dur global par page (défaut DEFAULT_RENDER_TIMEOUT_MS). */ + timeoutMs?: number + /** Délai injectable (tests instantanés). */ + delay?: (ms: number) => Promise +} + +const realDelay = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)) + +/** + * Rend UNE page dans un onglet arrière-plan et renvoie le Markdown de son DOM RENDU. + * + * Séquence : createTab (active:false) → waitForComplete → délai de rendu JS → + * EXTRACT_MARKDOWN → collecte des liens → removeTab. L'onglet est créé EN PREMIER (op. + * rapide) pour que son id soit toujours connu avant les étapes lentes : le finally peut + * alors le fermer quoi qu'il arrive. Les étapes lentes courent contre un timeout dur + * global ; ne rejette jamais (renvoie une raison discriminée). + */ +export async function renderPageInTab( + url: string, + deps: TabRenderDeps, +): Promise { + const renderDelayMs = deps.renderDelayMs ?? DEFAULT_RENDER_DELAY_MS + const timeoutMs = deps.timeoutMs ?? DEFAULT_RENDER_TIMEOUT_MS + const delay = deps.delay ?? realDelay + + let tabId: number | undefined + let timer: ReturnType | undefined + try { + // Onglet ouvert EN PREMIER : `tabId` est ainsi connu avant l'attente longue, donc le + // finally sait toujours quel onglet fermer (aucun orphelin sur timeout/erreur). + tabId = await deps.createTab(url) + const id = tabId + + // Étapes lentes isolées + `.catch` de secours : si le timeout gagne la course, cette + // promesse est ABANDONNÉE (l'onglet est déjà fermé par le finally) ; le catch évite + // qu'un extract/collectLinks sur un onglet fermé ne finisse en rejection non gérée. + const work: Promise = (async (): Promise => { + await deps.waitForComplete(id) + await delay(renderDelayMs) // laisse le JS de la SPA finir de peindre le DOM + const markdown = await deps.extractMarkdown(id) + if (!markdown) return { ok: false, reason: 'empty' } + const links = await deps.collectLinks(id).catch((): string[] => []) + return { ok: true, markdown, links } + })().catch((): RenderPageResponse => ({ ok: false, reason: 'network' })) + + // Timeout dur global : un onglet qui ne charge/rend jamais ne doit pas figer le crawl. + const timeout = new Promise((resolve) => { + timer = setTimeout(() => resolve({ ok: false, reason: 'timeout' }), timeoutMs) + }) + + return await Promise.race([work, timeout]) + } catch { + // createTab a échoué : aucun onglet ouvert (tabId undefined) → rien à fermer. + return { ok: false, reason: 'network' } + } finally { + if (timer) clearTimeout(timer) + // Nettoyage GARANTI : ferme l'onglet même sur timeout/erreur. + if (tabId !== undefined) await deps.removeTab(tabId).catch(() => {}) + } +} diff --git a/src/lib/url.ts b/src/lib/url.ts new file mode 100644 index 0000000..dea5e0e --- /dev/null +++ b/src/lib/url.ts @@ -0,0 +1,36 @@ +// PixelLens — URL helpers +// +// Small, dependency-free utilities for comparing where a scan/markdown result +// came from against the tab currently in front of the user, so the panel never +// presents another site's data as if it belonged to the active page. + +// Normalise a URL to its bare hostname (drops a leading `www.`) for same-site +// comparisons. Returns null when the input isn't a parseable http(s) URL — e.g. +// an empty string, or a `chrome://`/extension page — so callers can treat the +// host as "unknown" rather than mistaking it for a real mismatch. +export function getHost(url: string | null | undefined): string | null { + if (!url) return null + try { + const parsed = new URL(url) + // Only http(s) pages are scannable; treat chrome://, about:, file:, + // extension pages, etc. as "unknown" so the UI never flags a mismatch (or + // offers a re-scan) on a page it can't act on. + if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') return null + return parsed.hostname.replace(/^www\./, '') || null + } catch { + return null + } +} + +// True only when BOTH hosts are known AND differ. An unknown host on either +// side yields false, so the UI never flags a false "stale scan" mismatch (for +// instance when the active tab's URL can't be read without host permission). +export function isDifferentHost( + a: string | null | undefined, + b: string | null | undefined, +): boolean { + const ha = getHost(a) + const hb = getHost(b) + if (ha === null || hb === null) return false + return ha !== hb +} diff --git a/src/sidepanel/App.tsx b/src/sidepanel/App.tsx index b5970f7..08f759e 100644 --- a/src/sidepanel/App.tsx +++ b/src/sidepanel/App.tsx @@ -14,6 +14,7 @@ import gsap from 'gsap' import { usePanelStore, type PanelMode } from './store' import { MessageType } from '@/types/messages' import { getDesignSystems } from '@/lib/storage' +import { getHost } from '@/lib/url' import InspectorView from './views/InspectorView' import ScanView from './views/ScanView' import DesignSystemView from './views/DesignSystemView' @@ -60,6 +61,9 @@ function App() { const setCrawlResult = usePanelStore((s) => s.setCrawlResult) const setCrawlRunning = usePanelStore((s) => s.setCrawlRunning) const setCrawlExportMode = usePanelStore((s) => s.setCrawlExportMode) + const setActiveTabUrl = usePanelStore((s) => s.setActiveTabUrl) + const activeTabUrl = usePanelStore((s) => s.activeTabUrl) + const history = usePanelStore((s) => s.history) const tabsRef = useRef<(HTMLButtonElement | null)[]>([]) const indicatorRef = useRef(null) @@ -121,6 +125,60 @@ function App() { } }, [hasHydrated]) + // Track the URL of the tab currently in front of the user so the views can + // tell whether a restored scan/markdown belongs to the page on screen (see + // StaleScanBanner). Reading tab.url relies on the activeTab grant from the + // action click that opened the panel; a tab the extension has no host access + // to reports url=undefined, in which case we store null (unknown) and the + // views won't flag a false mismatch. Runs on mount, then follows tab switches + // and in-tab navigations (incl. SPA route changes) while the panel stays open. + useEffect(() => { + let cancelled = false + const readActiveTab = async () => { + try { + const [tab] = await chrome.tabs.query({ active: true, currentWindow: true }) + if (!cancelled) setActiveTabUrl(tab?.url ?? null) + } catch { + if (!cancelled) setActiveTabUrl(null) + } + } + void readActiveTab() + const onActivated = () => void readActiveTab() + const onUpdated = ( + _tabId: number, + changeInfo: chrome.tabs.OnUpdatedInfo, + tab: chrome.tabs.Tab, + ) => { + // React only to a URL change on the *active* tab — a full navigation or an + // SPA route push both surface here. Ignore background-tab churn. + if (tab.active && changeInfo.url) void readActiveTab() + } + chrome.tabs.onActivated.addListener(onActivated) + chrome.tabs.onUpdated.addListener(onUpdated) + return () => { + cancelled = true + chrome.tabs.onActivated.removeListener(onActivated) + chrome.tabs.onUpdated.removeListener(onUpdated) + } + }, [setActiveTabUrl]) + + // Reconcile the shown design system with the active tab. When we know the host + // on screen, prefer the most recent scan of THAT host — so switching to (or + // reopening on) a previously scanned site restores its scan instead of + // whatever was scanned last globally. If nothing matches we leave the current + // scan in place and the stale-scan banner flags the host mismatch. Skipped + // while the active URL is unknown, so it can never wrongly override the scan. + useEffect(() => { + if (!hasHydrated) return + const activeHost = getHost(activeTabUrl) + if (!activeHost) return + const current = usePanelStore.getState().designSystem + // Already showing a scan of the active host → nothing to do. + if (current && getHost(current.metadata.url) === activeHost) return + const match = history.find((d) => getHost(d.metadata.url) === activeHost) + if (match) usePanelStore.getState().setDesignSystem(match) + }, [hasHydrated, activeTabUrl, history]) + // GSAP sliding indicator. `hasHydrated` is a dependency so the indicator is // (re)positioned on the very paint where the restored activeMode lands, not a // stale default — and never left stuck on the wrong tab after rehydration. diff --git a/src/sidepanel/components/StaleScanBanner.tsx b/src/sidepanel/components/StaleScanBanner.tsx new file mode 100644 index 0000000..dc3d662 --- /dev/null +++ b/src/sidepanel/components/StaleScanBanner.tsx @@ -0,0 +1,44 @@ +import { WarningCircle, ArrowClockwise } from '@phosphor-icons/react' +import { getHost } from '@/lib/url' + +interface StaleScanBannerProps { + /** URL the currently displayed data was captured from. */ + scanUrl: string | null | undefined + /** Re-run the scan / conversion for the page currently in front of the user. */ + onRescan: () => void + /** Action verb for the button, e.g. "Scan" (default) or "Convert". */ + action?: string +} + +// Non-destructive notice shown at the top of a view when the result on screen +// was captured from a different site than the tab the user is on. It never wipes +// the data — the user keeps the previous result and can one-click re-run for the +// current page, or navigate back to the original site to see it unflagged. +export default function StaleScanBanner({ + scanUrl, + onRescan, + action = 'Scan', +}: StaleScanBannerProps) { + const host = getHost(scanUrl) ?? 'another page' + return ( +
+ +

+ Showing a previous scan of{' '} + {host} — not the page + you're on. +

+ +
+ ) +} diff --git a/src/sidepanel/store.ts b/src/sidepanel/store.ts index 0c41a30..f2787d8 100644 --- a/src/sidepanel/store.ts +++ b/src/sidepanel/store.ts @@ -54,6 +54,12 @@ interface PanelState { // default state first and then snaps to the restored one (see App.tsx). // Never persisted (excluded from `partialize`) — purely runtime state. hasHydrated: boolean + // URL of the tab currently in front of the user, tracked live from + // chrome.tabs (see App.tsx). Views compare it against the host their scan / + // markdown came from, so a restored result from another site is flagged as + // stale instead of shown as current. `null` = unknown (couldn't read the tab + // URL, e.g. no host permission for it). Never persisted — purely runtime. + activeTabUrl: string | null setMode: (mode: PanelMode) => void setInspectedElement: (el: InspectedElement | null) => void @@ -74,6 +80,7 @@ interface PanelState { setCrawlExportMode: (mode: CrawlExportMode) => void setCrawlRespectRobots: (respect: boolean) => void setHasHydrated: (hydrated: boolean) => void + setActiveTabUrl: (url: string | null) => void } export const usePanelStore = create()( @@ -96,6 +103,7 @@ export const usePanelStore = create()( crawlExportMode: 'single', crawlRespectRobots: true, hasHydrated: false, + activeTabUrl: null, setMode: (mode) => set({ activeMode: mode }), setInspectedElement: (el) => set({ inspectedElement: el }), @@ -117,6 +125,7 @@ export const usePanelStore = create()( setCrawlExportMode: (mode) => set({ crawlExportMode: mode }), setCrawlRespectRobots: (respect) => set({ crawlRespectRobots: respect }), setHasHydrated: (hydrated) => set({ hasHydrated: hydrated }), + setActiveTabUrl: (url) => set({ activeTabUrl: url }), }), { name: PANEL_STATE_STORAGE_KEY, diff --git a/src/sidepanel/views/CrawlView.tsx b/src/sidepanel/views/CrawlView.tsx index e4c3b6e..596acb4 100644 --- a/src/sidepanel/views/CrawlView.tsx +++ b/src/sidepanel/views/CrawlView.tsx @@ -12,6 +12,8 @@ import { Prohibit, Database, TreeStructure, + Lightning, + Browser, } from '@phosphor-icons/react' import { usePanelStore } from '../store' import { sendMessage } from '@/lib/messaging' @@ -52,6 +54,49 @@ function formatSkipReason(reason: string): string { } } +// Fast (fetch server HTML) vs Full (navigate each page in a real background tab so the +// JS runs, then capture the RENDERED DOM). Full is the fix for JS-rendered SPAs whose +// initial HTML is an empty shell. Local UI state — passed in the CRAWL_SITE payload. +function RenderModeToggle({ + value, + onChange, +}: { + value: 'fast' | 'full' + onChange: (v: 'fast' | 'full') => void +}) { + return ( +
+ {( + [ + { value: 'fast', label: 'Fast', icon: Lightning }, + { value: 'full', label: 'Full', icon: Browser }, + ] as const + ).map(({ value: v, label, icon: Icon }) => { + const isActive = value === v + return ( + + ) + })} +
+ ) +} + function CrawlView() { const running = usePanelStore((s) => s.crawlRunning) const progress = usePanelStore((s) => s.crawlProgress) @@ -66,16 +111,27 @@ function CrawlView() { const respectRobots = usePanelStore((s) => s.crawlRespectRobots) const setRespectRobots = usePanelStore((s) => s.setCrawlRespectRobots) const [copied, setCopied] = useState(false) + // Render mode is LOCAL (not persisted in the store): Fast (fetch) vs Full (real tab). + const [renderMode, setRenderMode] = useState<'fast' | 'full'>('fast') + // Mode the currently-shown result was crawled with — drives the "empty → try Full" hint. + const [resultMode, setResultMode] = useState<'fast' | 'full'>('fast') const handleStart = useCallback(async () => { setRunning(true) setError(null) setResult(null) setProgress(null) + // Fige le mode utilisé pour CE crawl : le résultat qui reviendra sera jugé (hint + // empty→Full) selon ce mode, même si l'utilisateur bascule le toggle entre-temps. + setResultMode(renderMode) try { // startUrl vide : le content script fait autorité et utilise location.href de // l'onglet courant (le panel n'a pas forcément accès à tab.url). - const res = await sendMessage(MessageType.CRAWL_SITE, { startUrl: '', respectRobots }) + const res = await sendMessage(MessageType.CRAWL_SITE, { + startUrl: '', + respectRobots, + renderMode, + }) if (!res?.success) { setRunning(false) setError('unsupported') @@ -85,7 +141,7 @@ function CrawlView() { setRunning(false) setError('failed') } - }, [setRunning, setError, setResult, setProgress, respectRobots]) + }, [setRunning, setError, setResult, setProgress, respectRobots, renderMode]) const handleStop = useCallback(() => { // Annulation coopérative : le crawl finit la page en cours puis renvoie le document @@ -189,8 +245,14 @@ function CrawlView() { > Crawl entire site +
+ Render + +

- {"Reads each page's initial HTML — client-rendered SPAs may come out partial."} + {renderMode === 'fast' + ? "Fast reads each page's initial HTML — client-rendered SPAs may come out partial." + : 'Full opens each page in a real background tab so the JS runs, then captures the rendered DOM. Slower, but handles SPAs.'}

@@ -206,6 +268,8 @@ function CrawlView() { .filter(([, n]) => n > 0) .sort((a, b) => b[1] - a[1]) const robotsSkipped = result.skippedReasons.robots ?? 0 + // Empty conversions in FAST mode are the signal that the site is JS-rendered → suggest Full. + const emptySkipped = result.skippedReasons.empty ?? 0 const preview = result.document.length > PREVIEW_LIMIT ? result.document.slice(0, PREVIEW_LIMIT) + @@ -293,6 +357,14 @@ function CrawlView() {

)} + {resultMode === 'fast' && emptySkipped > 0 && ( +

+ {emptySkipped.toLocaleString()} empty page{emptySkipped === 1 ? '' : 's'} — this site + renders with JavaScript. Switch “Render” to Full{' '} + below and crawl again. +

+ )} + {stats.pageCount === 0 && stats.skippedCount === 0 && (

No pages could be converted — the site may be client-rendered (its initial HTML @@ -308,6 +380,15 @@ function CrawlView() { {/* Actions */}

+ {/* Render mode — Fast (fetch) vs Full (real browser tab). Takes effect on the next + crawl (Crawl again). Full is the fix for JS-rendered SPAs. */} +
+ Render +
+ +
+
+ {/* Respect robots.txt — ON by default (safe). Turn OFF to include pages that robots.txt blocks: this is a human converting pages they can already see, not an indexing crawler. Takes effect on the next crawl (Crawl again). */} diff --git a/src/sidepanel/views/DesignSystemView.tsx b/src/sidepanel/views/DesignSystemView.tsx index d7def09..2f47559 100644 --- a/src/sidepanel/views/DesignSystemView.tsx +++ b/src/sidepanel/views/DesignSystemView.tsx @@ -4,12 +4,15 @@ import { usePanelStore } from '../store' import ExportButton from '../components/ExportButton' import ColorSwatch from '../components/ColorSwatch' import ShadowPreview from '../components/ShadowPreview' +import StaleScanBanner from '../components/StaleScanBanner' +import { isDifferentHost } from '@/lib/url' import type { DesignSystem } from '@/types/design-system' function DesignSystemView() { const designSystem = usePanelStore((s) => s.designSystem) const setDesignSystem = usePanelStore((s) => s.setDesignSystem) const setMode = usePanelStore((s) => s.setMode) + const activeTabUrl = usePanelStore((s) => s.activeTabUrl) if (!designSystem) { return ( @@ -49,8 +52,15 @@ function DesignSystemView() { setDesignSystem(updated) } + // Flag a design system restored from another site — the canonical scan CTA + // (with progress + error handling) lives in ScanView, so re-scan routes there. + const stale = isDifferentHost(designSystem.metadata.url, activeTabUrl) + return (
+ {stale && ( + setMode('scan')} /> + )} {/* Header with export */}
diff --git a/src/sidepanel/views/MarkdownView.tsx b/src/sidepanel/views/MarkdownView.tsx index 9405360..487188b 100644 --- a/src/sidepanel/views/MarkdownView.tsx +++ b/src/sidepanel/views/MarkdownView.tsx @@ -23,6 +23,8 @@ import { copyToClipboard, downloadMarkdown } from '@/lib/export' import { buildLlmBundle } from '@/lib/llm-bundle' import type { MarkdownFrontmatter, MarkdownResult } from '@/types/markdown' import MarkdownPreview from '../components/MarkdownPreview' +import StaleScanBanner from '../components/StaleScanBanner' +import { isDifferentHost } from '@/lib/url' // Build a safe .md filename from the page title (fallback: hostname). function slugify(value: string): string { @@ -62,6 +64,7 @@ function MarkdownView() { const setLoading = usePanelStore((s) => s.setMarkdownLoading) const setError = usePanelStore((s) => s.setMarkdownError) const designSystem = usePanelStore((s) => s.designSystem) + const activeTabUrl = usePanelStore((s) => s.activeTabUrl) const [copied, setCopied] = useState(false) const [copiedLlm, setCopiedLlm] = useState(false) @@ -191,8 +194,15 @@ function MarkdownView() { setTimeout(() => setCopiedLlm(false), 1500) } + // The persisted markdown result may be from a previously visited site; flag + // it rather than presenting it as the current page's conversion. + const stale = isDifferentHost(fm.url, activeTabUrl) + return (
+ {stale && ( + + )} {/* Meta header — frontmatter + stats */}
diff --git a/src/sidepanel/views/ScanView.tsx b/src/sidepanel/views/ScanView.tsx index 363364f..7b0f698 100644 --- a/src/sidepanel/views/ScanView.tsx +++ b/src/sidepanel/views/ScanView.tsx @@ -5,10 +5,12 @@ import { prefersReducedMotion } from '../reducedMotion' import { usePanelStore } from '../store' import { sendMessage } from '@/lib/messaging' import { MessageType } from '@/types/messages' +import { isDifferentHost } from '@/lib/url' import ColorPalette from '../components/ColorPalette' import TypeSpecimen from '../components/TypeSpecimen' import SpacingScale from '../components/SpacingScale' import ShadowPreview from '../components/ShadowPreview' +import StaleScanBanner from '../components/StaleScanBanner' type ScanTab = 'colors' | 'fonts' | 'spacing' | 'shadows' @@ -25,6 +27,7 @@ function ScanView() { const designSystem = usePanelStore((s) => s.designSystem) const setMode = usePanelStore((s) => s.setMode) const setScanError = usePanelStore((s) => s.setScanError) + const activeTabUrl = usePanelStore((s) => s.activeTabUrl) const [activeTab, setActiveTab] = useState('colors') const tabsRef = useRef<(HTMLButtonElement | null)[]>([]) const [indicatorStyle, setIndicatorStyle] = useState({ left: 0, width: 0 }) @@ -200,8 +203,16 @@ function ScanView() { } } + // The shown design system is the last scan globally; if that came from a + // different host than the tab on screen, flag it instead of passing it off + // as the current page. + const stale = isDifferentHost(designSystem.metadata.url, activeTabUrl) + return (
+ {stale && ( + + )} {/* Scan result tabs */}