diff --git a/CLAUDE.md b/CLAUDE.md index c6e054f..51180c3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -51,6 +51,7 @@ Prefer the preview servers in `.claude/launch.json` (`dev`, `showcase`) over ad- - `seal [--cost-usd N ...]` — import, validate gates, Playwright capture, verify contract, freeze the phase - `score ` — first call builds the isolated, blind-handled evaluator workspace (artifact copy + rubric + specs + template); second call (after the evaluator fills `assessment.json`) reruns gates against an out-of-repo copy and registers the evaluation. Panels: extra `assessment..json` files median-merge. `--sandbox` runs gates in Docker. - `status ` / `clean [--all]` / `export ` / `reindex` / `showcase` (publishes showcase build to `public/previews/showcase/`) +- `thumbs []` — regenerate the gallery's hover thumbnails (`public/previews//stage/thumb.jpg`) by shooting each published preview and cropping to the instrument chassis. Run it after publishing a new preview; the gallery derives the URL from `previewPath`, so a run without one simply shows no hover card. - **Transient workspaces (work/eval/gates) live under `~/.stagebench//`, not in the repo tree** — override with `STAGEBENCH_HOME`. Sealed `runs//stage/` never contains `node_modules` (gates run on a copy). ## Layout & architecture diff --git a/bench/cli.mjs b/bench/cli.mjs index b6945cd..4c97c4a 100644 --- a/bench/cli.mjs +++ b/bench/cli.mjs @@ -33,6 +33,7 @@ import { createEvalWorkspace, evalAssessmentPath, evalWorkspaceDir, removeEvalWo import { exportRun } from './lib/run/export.mjs' import { runChecks, verifyPhase } from './lib/run/verify.mjs' import { captureEvidence, serveDirectory } from './lib/run/capture.mjs' +import { generateThumbnails } from './lib/run/thumbnail.mjs' import { collectImplementationDetails } from './lib/implementation-details.mjs' import { loadRubric, runGates } from './lib/eval/evaluate.mjs' import { aggregateStageEvaluations, mergeAssessments, scoreAssessment } from './lib/eval/scoring.mjs' @@ -54,6 +55,7 @@ const COMMANDS = { clean: 'clean [] — remove transient work/eval/gates workspaces [--all] [--force to remove a running phase]', export: 'export [--out ] — bundle run.json, evaluations, report, and preview into a ZIP', showcase: 'Check, build, and publish showcase/ to public/previews/showcase', + thumbs: 'Regenerate gallery hover thumbnails from the published previews []', reindex: 'Regenerate src/data/runs.json from runs/*/run.json', fetch: 'Download the Nord manual and product photos into ./reference [--force] [--timeout ]', help: 'Show this help', @@ -380,6 +382,25 @@ try { fs.rmSync(destination, { recursive: true, force: true }) fs.cpSync(path.join(showcaseDir, 'dist'), destination, { recursive: true }) result = { published: '/previews/showcase/index.html', checks: checks.map(({ id, passed }) => ({ id, passed })) } + } else if (command === 'thumbs') { + // Hover thumbnails are derived from the published preview build, so the + // gallery can address one as previewPath with index.html swapped for + // thumb.jpg — no new field in the run record. + const locations = pathsFor(root) + const ids = id + ? [id] + : fs.readdirSync(path.join(root, 'runs')).filter((entry) => fs.existsSync(path.join(root, 'runs', entry, 'run.json'))) + const targets = [] + for (const runId of ids) { + const run = loadRun(root, runId) + if (!run.previewPath) continue + const stage = `stage${run.previewStage}` + const directory = path.join(locations.previews, run.id, stage) + if (!fs.existsSync(path.join(directory, 'index.html'))) continue + targets.push({ id: `${run.id}/${stage}`, directory, output: path.join(directory, 'thumb.jpg') }) + } + if (targets.length === 0) throw new Error('No published previews found to thumbnail') + result = { thumbnails: await generateThumbnails(targets) } } else if (command === 'reindex') { result = await reindexRegistry(root) } else if (command === 'fetch') { diff --git a/bench/lib/run/thumbnail.mjs b/bench/lib/run/thumbnail.mjs new file mode 100644 index 0000000..a54eea2 --- /dev/null +++ b/bench/lib/run/thumbnail.mjs @@ -0,0 +1,193 @@ +// Gallery hover thumbnails: shoot each published preview build and crop to the +// instrument itself, so the leaderboard shows the keyboard rather than whatever +// status text, help copy, and dead page background surrounds it. +// +// Every run lays the instrument out differently, so the crop is found in the +// DOM rather than hard-coded: locate the keyboard (the element with the most +// key-shaped children), then walk up to the painted chassis that contains it. +import fs from 'node:fs' +import path from 'node:path' +import { serveDirectory } from './capture.mjs' + +const VIEWPORT = { width: 1440, height: 900 } +// Wide enough to stay sharp on a HiDPI hover card without bloating the repo. +const THUMB_WIDTH = 720 +// Crop exactly to the chassis: even a few pixels of slack lets a status line +// sitting directly under the instrument bleed into the frame. +const PADDING = 0 + +// Runs in the page. Returns a viewport-space rect for the instrument, or null +// when nothing keyboard-shaped turns up and the caller should fall back. +function findInstrumentRect() { + const viewportArea = window.innerWidth * window.innerHeight + // A chassis may legitimately fill most of the window, but an element that + // covers essentially all of it is the page wrapper, not the instrument. + const MAX_SHARE = 0.92 + + const rectOf = (element) => element.getBoundingClientRect() + const isVisible = (element) => { + const style = getComputedStyle(element) + if (style.display === 'none' || style.visibility === 'hidden') return false + if (Number(style.opacity) === 0) return false + const rect = rectOf(element) + return rect.width > 0 && rect.height > 0 + } + const isPainted = (element) => { + const style = getComputedStyle(element) + const background = style.backgroundColor + const transparent = !background || background === 'transparent' || /,\s*0\s*\)$/.test(background) + return !transparent || style.backgroundImage !== 'none' + } + // Piano keys: tall, narrow, and all siblings of one another. + const isKeyShaped = (element) => { + const rect = rectOf(element) + return rect.height > 24 && rect.width > 3 && rect.width < 90 && rect.height > rect.width + } + + const elements = Array.from(document.body.querySelectorAll('*')).filter(isVisible) + + let keyboard = null + let keyCount = 0 + for (const element of elements) { + const keys = Array.from(element.children).filter(isKeyShaped) + if (keys.length > keyCount) { + keyCount = keys.length + keyboard = element + } + } + // An octave's worth of keys is the floor for trusting the detection. + if (!keyboard || keyCount < 12) return null + + // Climb to the outermost painted ancestor that still reads as the instrument + // rather than the page — that is the chassis holding panel plus keyboard. + let chassis = keyboard + for (let node = keyboard; node && node !== document.body; node = node.parentElement) { + const rect = rectOf(node) + if (rect.width * rect.height > viewportArea * MAX_SHARE) break + if (isPainted(node)) chassis = node + } + + const rect = rectOf(chassis) + return { x: rect.x, y: rect.y, width: rect.width, height: rect.height, keyCount } +} + +// Union of every painted element that is not the page wrapper. Used when key +// detection fails, so a phase-1 shell still gets a sensible crop. +function findPaintedBounds() { + const viewportArea = window.innerWidth * window.innerHeight + let left = Infinity + let top = Infinity + let right = -Infinity + let bottom = -Infinity + for (const element of document.body.querySelectorAll('*')) { + const style = getComputedStyle(element) + if (style.display === 'none' || style.visibility === 'hidden' || Number(style.opacity) === 0) continue + const background = style.backgroundColor + const transparent = !background || background === 'transparent' || /,\s*0\s*\)$/.test(background) + if (transparent && style.backgroundImage === 'none') continue + const rect = element.getBoundingClientRect() + if (rect.width <= 0 || rect.height <= 0) continue + if (rect.width * rect.height > viewportArea * 0.92) continue + left = Math.min(left, rect.x) + top = Math.min(top, rect.y) + right = Math.max(right, rect.x + rect.width) + bottom = Math.max(bottom, rect.y + rect.height) + } + if (!Number.isFinite(left)) return null + return { x: left, y: top, width: right - left, height: bottom - top, keyCount: 0 } +} + +function clampToViewport(rect) { + const left = Math.max(0, Math.floor(rect.x - PADDING)) + const top = Math.max(0, Math.floor(rect.y - PADDING)) + const right = Math.min(VIEWPORT.width, Math.ceil(rect.x + rect.width + PADDING)) + const bottom = Math.min(VIEWPORT.height, Math.ceil(rect.y + rect.height + PADDING)) + return { x: left, y: top, width: Math.max(1, right - left), height: Math.max(1, bottom - top) } +} + +// Playwright cannot resize a screenshot, so the full-resolution crop is +// replayed into a blank page at the target width and reshot. Keeps thumbnails a +// uniform width with no image-processing dependency. +async function downscale(browser, buffer, clip) { + const height = Math.max(1, Math.round((clip.height / clip.width) * THUMB_WIDTH)) + const context = await browser.newContext({ + viewport: { width: THUMB_WIDTH, height }, + deviceScaleFactor: 1, + }) + const page = await context.newPage() + await page.setContent( + ``, + ) + await page.waitForTimeout(60) + const out = await page.screenshot({ type: 'jpeg', quality: 82 }) + await context.close() + return out +} + +async function shootOne(browser, { directory, output }) { + const server = await serveDirectory(directory) + const context = await browser.newContext({ + viewport: VIEWPORT, + deviceScaleFactor: 1, + colorScheme: 'light', + reducedMotion: 'reduce', + locale: 'en-US', + timezoneId: 'UTC', + }) + try { + const page = await context.newPage() + await page.goto(server.url, { waitUntil: 'networkidle', timeout: 60_000 }) + await page.evaluate(async () => document.fonts.ready) + await page.addStyleTag({ + content: '*,*::before,*::after{animation:none!important;transition:none!important;caret-color:transparent!important}', + }) + await page.waitForTimeout(120) + + let rect = await page.evaluate(findInstrumentRect) + let strategy = 'instrument' + if (!rect) { + rect = await page.evaluate(findPaintedBounds) + strategy = 'painted-bounds' + } + if (!rect) { + rect = { x: 0, y: 0, width: VIEWPORT.width, height: VIEWPORT.height, keyCount: 0 } + strategy = 'viewport' + } + const clip = clampToViewport(rect) + const full = await page.screenshot({ clip, animations: 'disabled' }) + const thumb = await downscale(browser, full, clip) + fs.mkdirSync(path.dirname(output), { recursive: true }) + fs.writeFileSync(output, thumb) + return { strategy, keyCount: rect.keyCount ?? 0, clip, bytes: thumb.length } + } finally { + await context.close() + await server.close() + } +} + +// Generates a thumbnail for every published preview named in the registry. +// `targets` is [{ id, directory, output }]. +export async function generateThumbnails(targets) { + let chromium + try { + ;({ chromium } = await import('playwright')) + } catch { + throw new Error('Playwright is required. Run pnpm install, then pnpm exec playwright install chromium.') + } + const browser = await chromium.launch({ headless: true }) + const results = [] + try { + for (const target of targets) { + try { + const result = await shootOne(browser, target) + results.push({ id: target.id, ok: true, ...result }) + } catch (error) { + results.push({ id: target.id, ok: false, error: error.message }) + } + } + } finally { + await browser.close() + } + return results +} diff --git a/public/previews/claude-fable-5/stage2/thumb.jpg b/public/previews/claude-fable-5/stage2/thumb.jpg new file mode 100644 index 0000000..b7bd9f3 Binary files /dev/null and b/public/previews/claude-fable-5/stage2/thumb.jpg differ diff --git a/public/previews/claude-haiku-4-5-20251001/stage3/thumb.jpg b/public/previews/claude-haiku-4-5-20251001/stage3/thumb.jpg new file mode 100644 index 0000000..d5df4c1 Binary files /dev/null and b/public/previews/claude-haiku-4-5-20251001/stage3/thumb.jpg differ diff --git a/public/previews/claude-opus-4-8/stage3/thumb.jpg b/public/previews/claude-opus-4-8/stage3/thumb.jpg new file mode 100644 index 0000000..a76cc88 Binary files /dev/null and b/public/previews/claude-opus-4-8/stage3/thumb.jpg differ diff --git a/public/previews/composer-2-5-fast/stage3/thumb.jpg b/public/previews/composer-2-5-fast/stage3/thumb.jpg new file mode 100644 index 0000000..a25677c Binary files /dev/null and b/public/previews/composer-2-5-fast/stage3/thumb.jpg differ diff --git a/public/previews/gpt-5-6-luna-3/stage3/thumb.jpg b/public/previews/gpt-5-6-luna-3/stage3/thumb.jpg new file mode 100644 index 0000000..e3854c7 Binary files /dev/null and b/public/previews/gpt-5-6-luna-3/stage3/thumb.jpg differ diff --git a/public/previews/gpt-5-6-sol-high/stage3/thumb.jpg b/public/previews/gpt-5-6-sol-high/stage3/thumb.jpg new file mode 100644 index 0000000..8c50398 Binary files /dev/null and b/public/previews/gpt-5-6-sol-high/stage3/thumb.jpg differ diff --git a/public/previews/gpt-5-6-terra-high/stage3/thumb.jpg b/public/previews/gpt-5-6-terra-high/stage3/thumb.jpg new file mode 100644 index 0000000..1df6063 Binary files /dev/null and b/public/previews/gpt-5-6-terra-high/stage3/thumb.jpg differ diff --git a/public/previews/gpt5-5-high/stage3/thumb.jpg b/public/previews/gpt5-5-high/stage3/thumb.jpg new file mode 100644 index 0000000..dc8e250 Binary files /dev/null and b/public/previews/gpt5-5-high/stage3/thumb.jpg differ diff --git a/public/previews/grok-4-5/stage3/thumb.jpg b/public/previews/grok-4-5/stage3/thumb.jpg new file mode 100644 index 0000000..7260f03 Binary files /dev/null and b/public/previews/grok-4-5/stage3/thumb.jpg differ diff --git a/public/previews/kimi-k3/stage3/thumb.jpg b/public/previews/kimi-k3/stage3/thumb.jpg new file mode 100644 index 0000000..e1eb68d Binary files /dev/null and b/public/previews/kimi-k3/stage3/thumb.jpg differ diff --git a/src/App.css b/src/App.css index 7bc1314..76daebb 100644 --- a/src/App.css +++ b/src/App.css @@ -996,6 +996,42 @@ a { color: inherit; } .row-main.is-playable:active { background: oklch(95% 0.008 29); } } +/* Hover thumbnail: a cropped shot of the run's instrument. It clears the + hovered row entirely — that row stays readable while its preview is up — and + lines its left edge up with the table's, so it reads as belonging to the + leaderboard rather than floating loose in the page margin. Pointer-only: + hidden below the 900px breakpoint, where the row stops being one grid line + and a floating card would have nowhere to sit. Never intercepts the pointer, + so the row underneath stays clickable through it. */ +.row-thumb { + display: none; + position: absolute; + left: 0; + z-index: 10; + /* Flush left, the width is what is left before the phase-sector columns — so + a card only ever covers neighbouring model names, never anyone's scores. */ + width: 420px; + padding: 6px; + border: 1px solid var(--line); + border-radius: 8px; + background: var(--white); + box-shadow: 0 18px 34px -14px oklch(30% 0.03 29 / .5), 0 3px 8px -3px oklch(30% 0.03 29 / .32); + opacity: 0; + pointer-events: none; + transition: opacity .14s ease, transform .14s ease; +} +.row-thumb.is-above { bottom: calc(100% + 6px); transform: translateY(5px); } +.row-thumb.is-below { top: calc(100% + 6px); transform: translateY(-5px); } +.row-thumb.is-loaded { opacity: 1; transform: translateY(0); } +.row-thumb img { display: block; width: 100%; height: auto; border-radius: 4px; } + +@media (hover: hover) and (min-width: 901px) { + .row-thumb { display: block; } +} +@media (prefers-reduced-motion: reduce) { + .row-thumb.is-above, .row-thumb.is-below, .row-thumb.is-loaded { transform: none; } +} + .driver { min-width: 0; } .driver strong { display: block; overflow: hidden; font-size: 15px; font-weight: 800; letter-spacing: -.01em; text-overflow: ellipsis; white-space: nowrap; } .driver > span { color: var(--muted); font: 500 9.5px/1.9 ui-monospace, monospace; } diff --git a/src/components/RunList.tsx b/src/components/RunList.tsx index 6086196..6fae161 100644 --- a/src/components/RunList.tsx +++ b/src/components/RunList.tsx @@ -3,10 +3,10 @@ // Memoized: all props are reference-stable, so App's dialog/viewer state // changes (open preview, copy link, phase switch) skip re-rendering the // largest subtree in the app. -import { Fragment, memo } from 'react' +import { Fragment, memo, useCallback, useState } from 'react' import type { CSSProperties } from 'react' import type { PhaseNumber } from '../run-utils' -import { floorScore, getRunTitle } from '../run-utils' +import { floorScore, getRunTitle, getThumbPath } from '../run-utils' import type { RunEntry } from '../types' import { bestByTierPhase, @@ -23,6 +23,11 @@ import { } from '../runs-data' import { ChevronIcon, InfoIcon, PlayIcon, ReportIcon, StatusLight } from './icons' +// Tallest a hover card gets (a near-square instrument crop at the card's width, +// plus its frame). A row closer than this to the top of the viewport shows its +// card below instead of above. +const THUMB_MAX_HEIGHT = 300 + export const RunList = memo(function RunList({ expandedRunId, onToggleExpand, @@ -36,6 +41,18 @@ export const RunList = memo(function RunList({ onOpenProtocolInfo: () => void onOpenReport: (run: RunEntry) => void }) { + // Pointer-only affordance: the hovered row shows a cropped shot of its + // instrument. Mounted on hover rather than up front so the leaderboard does + // not pull ten screenshots on load; `loaded` keeps the card hidden until its + // image has decoded, so it fades in whole instead of flashing an empty frame. + // The card normally sits above the row; rows too near the top of the viewport + // flip it below so it is never clipped off-screen. + const [hovered, setHovered] = useState<{ id: string; placement: 'above' | 'below' } | null>(null) + const [loadedThumbs, setLoadedThumbs] = useState>(() => new Set()) + const markThumbLoaded = useCallback((runId: string) => { + setLoadedThumbs((current) => (current.has(runId) ? current : new Set(current).add(runId))) + }, []) + if (visibleRuns.length === 0) { return (
@@ -53,6 +70,8 @@ export const RunList = memo(function RunList({ const phaseList = getPhaseList(run) const rank = rankByRun.get(run.id) ?? null const playable = Boolean(run.previewPath) + const thumbPath = getThumbPath(run) + const showThumb = playable && thumbPath !== null && hovered?.id === run.id const expanded = expandedRunId === run.id // Sector column count rides on a CSS variable so legacy tiers // with a four-phase protocol keep their own aligned grid. @@ -94,8 +113,21 @@ export const RunList = memo(function RunList({
onOpenPreview(run) : undefined} + onMouseEnter={playable ? (event) => { + const { top } = event.currentTarget.getBoundingClientRect() + setHovered({ id: run.id, placement: top < THUMB_MAX_HEIGHT ? 'below' : 'above' }) + } : undefined} + onMouseLeave={playable ? () => setHovered((current) => (current?.id === run.id ? null : current)) : undefined} style={sectorsStyle} > + {showThumb && ( + + )} {playable && ( diff --git a/src/run-utils-runtime.ts b/src/run-utils-runtime.ts index c0448d3..8935136 100644 --- a/src/run-utils-runtime.ts +++ b/src/run-utils-runtime.ts @@ -26,6 +26,15 @@ export function getPreviewPath(run: PreviewRun, phase: PhaseNumber): string | nu return undefined } +// Hover thumbnails sit beside the published preview build they were shot from, +// so they need no field of their own on the run record. Generated by +// `pnpm bench thumbs`; a run whose preview predates that command has none, and +// the gallery simply shows no hover card. +export function getThumbPath(run: PreviewRun): string | null { + if (!run.previewPath) return null + return run.previewPath.replace(/index\.html$/, 'thumb.jpg') +} + export function getAvailablePhases(run: PreviewRun): PhaseNumber[] { return ALL_PHASES.filter((phase) => Boolean(getPreviewPath(run, phase))) } diff --git a/src/run-utils.ts b/src/run-utils.ts index cf2bc1e..3bb56e2 100644 --- a/src/run-utils.ts +++ b/src/run-utils.ts @@ -10,6 +10,7 @@ export { getLatestPhase, getPreviewPath, getRunTitle, + getThumbPath, parseViewerSearch, } from './run-utils-runtime'