From 6c9f6338bd26052f624bc9b3f4c7fb3b37aad7be Mon Sep 17 00:00:00 2001 From: Matt Rubens <2600+mrubens@users.noreply.github.com> Date: Wed, 12 Aug 2026 15:42:06 -0400 Subject: [PATCH 1/3] [Proto] Annotation layer: capture-anchored highlight boxes and callouts Capture emits timeline.annotations from the full element rect (previously resolved and discarded); a note modifier on show beats scopes each annotation to its beat's caption window. New Annotations.tsx renders boxes and label chips inside the window transform with counter-scaled chrome. Prototype only: not validated beyond a wide-preset in-image render. --- .../standard/feature-demo/capture/capture.mjs | 35 +++++- .../feature-demo/render/src/Annotations.tsx | 108 ++++++++++++++++++ .../feature-demo/render/src/DemoStage.tsx | 17 +++ 3 files changed, 159 insertions(+), 1 deletion(-) create mode 100644 packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/Annotations.tsx diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/capture/capture.mjs b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/capture/capture.mjs index f70066690..06c60124a 100644 --- a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/capture/capture.mjs +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/capture/capture.mjs @@ -118,6 +118,17 @@ const centerNorm = (r) => ({ y: (r.y + r.h / 2) / VIEWPORT.h, }); +// Full normalized bounding box — the anchor for annotations. Resolved AFTER +// the beat's scroll settles, because rects are viewport-relative: an anchor +// is only valid for the scroll position it was measured at, which is why +// annotations live strictly inside their beat's window. +const boxNorm = (r) => ({ + x: r.x / VIEWPORT.w, + y: r.y / VIEWPORT.h, + w: r.w / VIEWPORT.w, + h: r.h / VIEWPORT.h, +}); + const timeline = { video: { path: 'recording.mp4', width: VIEWPORT.w, height: VIEWPORT.h }, fps: 30, @@ -125,6 +136,7 @@ const timeline = { cursorKeys: [{ t: 0, v: { x: 0.5, y: 1.1 } }], clicks: [], captions: [], + annotations: [], // Optional declarative caption styling from the demo script (position, // accent, pill, sizeScale); the renderer merges it over preset defaults. ...(script.captionStyle ? { captionStyle: script.captionStyle } : {}), @@ -195,6 +207,21 @@ async function run() { sleep(beat.settleMs ?? 600); // scroll settles on screen const settled = now(); + // `note` rides the beat: a highlight box (plus optional label chip) + // anchored to the beat's own element, shown for the beat's caption + // window. One per beat — it is a clarity device, not a diagram. + const noteBox = beat.note ? boxNorm(rect(beat.note.sel ?? beat.sel)) : null; + const pushNote = (start, end) => { + if (!noteBox) return; + timeline.annotations.push({ + start: Math.round(start * 1000) / 1000, + end: Math.round(end * 1000) / 1000, + box: noteBox, + ...(beat.note.text ? { text: beat.note.text } : {}), + style: beat.note.style ?? 'callout', + }); + }; + if (beat.caption) { const lineSeconds = narration ? narration.clips[lineIndex].durationSeconds @@ -215,10 +242,16 @@ async function run() { } lineIndex += 1; + // The note breathes with the caption: appears a beat after the line + // starts (the voice names the subject, then the box lands on it). + pushNote(lineStart + 0.35, lineEnd + 0.25); + const holdSeconds = Math.max(0.5, lineEnd + LINE_GAP - now()); sleep(holdSeconds * 1000); } else { - sleep(beat.holdMs ?? 900); + const holdMs = beat.holdMs ?? 900; + pushNote(settled + 0.15, settled + holdMs / 1000 + 0.6); + sleep(holdMs); } continue; } diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/Annotations.tsx b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/Annotations.tsx new file mode 100644 index 000000000..edac23434 --- /dev/null +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/Annotations.tsx @@ -0,0 +1,108 @@ +// Annotation layer: capture-anchored highlight boxes and callout chips drawn +// over the recording. Rendered INSIDE the window transform container (like +// the cursor and click ripple) so boxes stay glued to page coordinates, with +// chrome counter-scaled by 1/S so stroke widths and chip text keep constant +// on-screen size under a preset baseScale. +// +// Anchors are resolved at beat-settle time and are only valid for that +// scroll position, so each annotation lives strictly inside its beat's +// caption window — capture scopes the timing, the renderer only fades. + +type Box = { x: number; y: number; w: number; h: number }; + +export type Annotation = { + start: number; + end: number; + box: Box; + text?: string; + style?: 'box' | 'callout'; +}; + +const clamp = (v: number, lo: number, hi: number) => + Math.max(lo, Math.min(hi, v)); + +export const Annotations: React.FC<{ + annotations: Annotation[]; + t: number; + baseW: number; + baseH: number; + invScale: number; + accent: string; +}> = ({ annotations, t, baseW, baseH, invScale, accent }) => { + return ( + <> + {annotations.map((a, i) => { + const fadeIn = clamp((t - a.start) / 0.25, 0, 1); + const fadeOut = clamp((a.end - t) / 0.25, 0, 1); + const opacity = Math.min(fadeIn, fadeOut); + if (opacity <= 0) return null; + + const left = a.box.x * baseW; + const top = a.box.y * baseH; + const width = a.box.w * baseW; + const height = a.box.h * baseH; + + // Draw-on: the box eases from a slightly loose fit to snug as it + // fades in, so it reads as placed rather than popped. + const settle = 1 - fadeIn; + const inset = -6 - settle * 10; + + const showChip = a.style !== 'box' && a.text; + // Chip above the box unless that would leave the window; the gap is + // counter-scaled with the chip itself. + const chipAbove = top > 64 * invScale; + + return ( +
+
+ {showChip ? ( +
+
+ {a.text} +
+
+ ) : null} +
+ ); + })} + + ); +}; diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/DemoStage.tsx b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/DemoStage.tsx index 2c5794df0..509876489 100644 --- a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/DemoStage.tsx +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/DemoStage.tsx @@ -1,6 +1,7 @@ import { OffthreadVideo, staticFile, useCurrentFrame } from 'remotion'; import timeline from '../props/timeline.json'; import narration from '../props/narration.json'; +import { Annotations, type Annotation } from './Annotations'; import type { CaptionStyle } from './presets'; type WordTiming = { text: string; start: number; end: number }; @@ -262,6 +263,22 @@ export const DemoStage: React.FC<{
) : null} + + {/* Annotation layer: same page-coordinate space as the cursor and + ripple, chrome counter-scaled so it holds constant on-screen + size under a preset baseScale. Older timelines have no + annotations key and render unchanged. */} + From 50c25ab50a7b0373f66670115ec47b9619b3fa45 Mon Sep 17 00:00:00 2001 From: Matt Rubens <2600+mrubens@users.noreply.github.com> Date: Wed, 12 Aug 2026 15:58:03 -0400 Subject: [PATCH 2/3] [Feat] Spotlight annotations, tight anchors, docs, and tests Iterating on the prototype: the default style becomes spotlight (dim the window, leave the target bright) after outline boxes over dense real pages read as stickers; anchors tighten to the content rect via a Range so block elements do not produce full-column boxes; the label chip sits on the dimmed area, width-clamped inside the window. SKILL.md documents the one-note-per-beat grammar, render/README replaces the stale edge-clamp bullet (it predated the unconditional clamps) and records the new window-clip invariant, and the skill test asserts the contract. Validated in the worker image against the live self-hosting docs page: both presets rendered from one capture; a top-strip probe confirms the dim never reaches the backdrop (max channel delta 1 wide / 0 vertical between annotated and unannotated moments). --- .../__tests__/featureDemoSkill.test.ts | 25 ++++ .../skills/standard/feature-demo/SKILL.md | 1 + .../standard/feature-demo/capture/capture.mjs | 28 +++- .../standard/feature-demo/render/README.md | 23 +++- .../feature-demo/render/src/Annotations.tsx | 123 +++++++++++------- 5 files changed, 147 insertions(+), 53 deletions(-) diff --git a/packages/cloud-agents/src/server/workflows/__tests__/featureDemoSkill.test.ts b/packages/cloud-agents/src/server/workflows/__tests__/featureDemoSkill.test.ts index b80485b94..759ce3441 100644 --- a/packages/cloud-agents/src/server/workflows/__tests__/featureDemoSkill.test.ts +++ b/packages/cloud-agents/src/server/workflows/__tests__/featureDemoSkill.test.ts @@ -132,6 +132,31 @@ describe('feature-demo skill', () => { expect(captureRunner).toContain('pushCursorMove(moveStart, moveEnd, c);'); }); + it('anchors annotations to capture-measured element rects', () => { + const captureRunner = fs.readFileSync( + path.join(skillDirPath, 'capture/capture.mjs'), + 'utf8', + ); + const annotationsComponent = fs.readFileSync( + path.join(skillDirPath, 'render/src/Annotations.tsx'), + 'utf8', + ); + + // Capture measures the anchor (tight content rect, resolved after the + // beat's scroll settles) and scopes it to the beat's caption window; + // the renderer only draws at capture-emitted rects. + expect(captureRunner).toContain('annotations: []'); + expect(captureRunner).toContain('function tightRect('); + expect(captureRunner).toContain('selectNodeContents'); + expect(captureRunner).toContain("beat.note.style ?? 'spotlight'"); + // The layer is clipped to the window so a spotlight dim can never touch + // the backdrop or the caption band. + expect(annotationsComponent).toContain("overflow: 'hidden'"); + // Documented as a rare, single-per-beat clarity device. + expect(skillContent).toContain('ONE `"note"`'); + expect(skillContent).toContain('not decoration'); + }); + it.each(['focus', 'reset'])('rejects the unsupported %s action', (action) => { const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'feature-demo-')); const scriptPath = path.join(tempDir, 'demo-script.json'); diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md index 501796f47..938d62114 100644 --- a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md @@ -57,6 +57,7 @@ this pipeline is ever committed to the repository. Write `/tmp/feature-demo/demo-script.json`: `{ "url": string, "viewport": { "w": 1280, "h": 800 }, "beats": [...] }`. Beat actions: - `{ "a": "show", "sel": css, "caption": "..." }` — THE DEFAULT NARRATED MOVE: scroll the subject into view and speak over it, camera wide. No cursor. +- A `show` beat may carry ONE `"note"` — `{ "sel": css?, "text": "..."?, "style": "spotlight"|"box"|"callout"? }` — an attention cue anchored to a real element for exactly that beat's caption window. Default `spotlight` dims the rest of the page and leaves the target bright, with `text` as a small label on the dimmed area; use it when the narration names one thing the viewer should find (`note.sel` may point at a more specific element than the beat scrolls to, e.g. the command block under a heading). `box`/`callout` draw an accent outline instead — busier on dense pages, so prefer spotlight. At most one note per beat, and most beats want none: it is a clarity device, not decoration. - `{ "a": "wait", "ms": n }` / `{ "a": "hold", "ms": n }` — let the page settle / linger. - `{ "a": "scrollTo", "sel": css, "ms": n }` — plain scroll with no narration attached. - `{ "a": "click", "sel": css, "holdMs": ~300 }` — real click with ripple. diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/capture/capture.mjs b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/capture/capture.mjs index 06c60124a..bb052433f 100644 --- a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/capture/capture.mjs +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/capture/capture.mjs @@ -129,6 +129,28 @@ const boxNorm = (r) => ({ h: r.h / VIEWPORT.h, }); +// Tight content rect for annotation anchors. Block elements (headings, +// paragraphs) report full-column boxes with dead space past the text; a +// Range over the contents hugs what the viewer actually reads. Falls back +// to the element box for empty/replaced elements. +function tightRect(sel) { + const selB64 = Buffer.from(String(sel), 'utf8').toString('base64'); + if (!/^[A-Za-z0-9+/=]*$/.test(selB64)) { + throw new Error(`unencodable selector: ${sel}`); + } + const js = + `(function(){var e=document.querySelector(atob("${selB64}"));` + + `if(!e)return null;var r=e.getBoundingClientRect();` + + `try{var g=document.createRange();g.selectNodeContents(e);` + + `var t=g.getBoundingClientRect();` + + `if(t&&t.width>1&&t.height>1)r=t;}catch(_){}` + + `return{x:r.x,y:r.y,w:r.width,h:r.height};})()`; + const out = ab('eval', js).trim(); + const r = JSON.parse(out); + if (!r) throw new Error(`element not found: ${sel}`); + return r; +} + const timeline = { video: { path: 'recording.mp4', width: VIEWPORT.w, height: VIEWPORT.h }, fps: 30, @@ -210,7 +232,9 @@ async function run() { // `note` rides the beat: a highlight box (plus optional label chip) // anchored to the beat's own element, shown for the beat's caption // window. One per beat — it is a clarity device, not a diagram. - const noteBox = beat.note ? boxNorm(rect(beat.note.sel ?? beat.sel)) : null; + const noteBox = beat.note + ? boxNorm(tightRect(beat.note.sel ?? beat.sel)) + : null; const pushNote = (start, end) => { if (!noteBox) return; timeline.annotations.push({ @@ -218,7 +242,7 @@ async function run() { end: Math.round(end * 1000) / 1000, box: noteBox, ...(beat.note.text ? { text: beat.note.text } : {}), - style: beat.note.style ?? 'callout', + style: beat.note.style ?? 'spotlight', }); }; diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/README.md b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/README.md index c9d1e58ac..0ac94e239 100644 --- a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/README.md +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/README.md @@ -10,13 +10,22 @@ render those; adaptations change how they look, not what they mean. Worth preserving when adapting — each encodes a bug class found the hard way: -- **Counter-scaled cursor/ripple** (`DemoStage.tsx`): cursor and ripple are - children of the zoom-transform container with a `1/S` counter-scale, so - they stay glued to page coordinates at constant size through any zoom. -- **Edge-clamp guard** (`DemoStage.tsx`): the translate that keeps backdrop - from showing behind the window must only clamp an axis when the scaled - window exceeds the canvas on that axis; clamping a smaller-than-canvas - window (vertical presets) shoves it into a corner. +- **Counter-scaled cursor/ripple/annotations** (`DemoStage.tsx`, + `Annotations.tsx`): overlays are children of the window transform container + with a `1/S` counter-scale on their chrome, so they stay glued to page + coordinates at constant on-screen size under a preset `baseScale`. +- **Unconditional edge clamps** (`DemoStage.tsx`): the same pair of translate + bounds means "keep covering the stage" when the scaled window exceeds it + and "stay inside the stage" when it does not — the two just swap order — + so clamping to their min/max is correct in both regimes. Do not gate the + clamps on window size; the gated version let a nearly-stage-height window + drift into the caption band. +- **Window-clipped annotations** (`Annotations.tsx`): the annotation layer is + clipped to the window rect (same 16px radius as the video panel) so a + spotlight dim covers exactly the recording — never the backdrop or the + caption band — and each annotation lives strictly inside its beat's + caption window, because its anchor rect is only valid for the scroll + position it was measured at. - **Timeline-driven interpolation**: all motion eases between capture-emitted keys. Do not invent motion that is not in the timeline — it will drift from the recording. diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/Annotations.tsx b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/Annotations.tsx index edac23434..780f5df5d 100644 --- a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/Annotations.tsx +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/Annotations.tsx @@ -1,9 +1,14 @@ -// Annotation layer: capture-anchored highlight boxes and callout chips drawn -// over the recording. Rendered INSIDE the window transform container (like -// the cursor and click ripple) so boxes stay glued to page coordinates, with +// Annotation layer: capture-anchored attention cues drawn over the +// recording. Rendered INSIDE the window transform container (like the +// cursor and click ripple) so anchors stay glued to page coordinates, with // chrome counter-scaled by 1/S so stroke widths and chip text keep constant // on-screen size under a preset baseScale. // +// The default style is `spotlight`: dim the rest of the window and leave +// the target bright. Attention comes from contrast, not from chrome drawn +// onto the page — boxes and labels over a dense real page read as stickers, +// which is why the outline styles are opt-in. +// // Anchors are resolved at beat-settle time and are only valid for that // scroll position, so each annotation lives strictly inside its beat's // caption window — capture scopes the timing, the renderer only fades. @@ -15,7 +20,7 @@ export type Annotation = { end: number; box: Box; text?: string; - style?: 'box' | 'callout'; + style?: 'spotlight' | 'box' | 'callout'; }; const clamp = (v: number, lo: number, hi: number) => @@ -30,54 +35,84 @@ export const Annotations: React.FC<{ accent: string; }> = ({ annotations, t, baseW, baseH, invScale, accent }) => { return ( - <> + // Clip to the window rect (same radius as the video panel) so the + // spotlight dim covers exactly the recording, never the backdrop. +
{annotations.map((a, i) => { - const fadeIn = clamp((t - a.start) / 0.25, 0, 1); - const fadeOut = clamp((a.end - t) / 0.25, 0, 1); + const fadeIn = clamp((t - a.start) / 0.35, 0, 1); + const fadeOut = clamp((a.end - t) / 0.35, 0, 1); const opacity = Math.min(fadeIn, fadeOut); if (opacity <= 0) return null; - const left = a.box.x * baseW; - const top = a.box.y * baseH; - const width = a.box.w * baseW; - const height = a.box.h * baseH; - - // Draw-on: the box eases from a slightly loose fit to snug as it - // fades in, so it reads as placed rather than popped. - const settle = 1 - fadeIn; - const inset = -6 - settle * 10; + // Breathing room around the measured content rect. + const padX = 14; + const padY = 10; + const left = a.box.x * baseW - padX; + const top = a.box.y * baseH - padY; + const width = a.box.w * baseW + 2 * padX; + const height = a.box.h * baseH + 2 * padY; - const showChip = a.style !== 'box' && a.text; - // Chip above the box unless that would leave the window; the gap is - // counter-scaled with the chip itself. - const chipAbove = top > 64 * invScale; + const spotlight = a.style !== 'box' && a.style !== 'callout'; + const showChip = a.text && a.style !== 'box'; + // Chip sits on the dimmed area above the cutout, left-aligned with + // it (or below when the target is near the top edge). Its rendered + // width is estimated from the label (nowrap, so width tracks text) + // to keep the right edge inside the window. + const chipAbove = top > 72 * invScale; + const chipW = ((a.text?.length ?? 0) * 17 * 0.56 + 28) * invScale; + const chipLeft = clamp(left, 12, Math.max(12, baseW - 12 - chipW)); return (
-
+ {spotlight ? ( + // The cutout: a transparent rounded rect whose enormous + // box-shadow dims everything else in the (clipped) window. +
+ ) : ( +
+ )} {showChip ? (
{a.text} @@ -103,6 +138,6 @@ export const Annotations: React.FC<{
); })} - +
); }; From cb3a2a1ea4069ea80a3efb725c19dd8901283687 Mon Sep 17 00:00:00 2001 From: Matt Rubens <2600+mrubens@users.noreply.github.com> Date: Wed, 12 Aug 2026 16:07:20 -0400 Subject: [PATCH 3/3] [Fix] End captionless annotations at the hold boundary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The next beat may scroll the moment the hold ends, and the anchor rect is only valid for its own scroll position — a 0.6s tail dimmed a stale rectangle over the next screen. The renderer's fade-out completes inside the hold. --- .../skills/standard/feature-demo/capture/capture.mjs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/capture/capture.mjs b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/capture/capture.mjs index bb052433f..b5fea086c 100644 --- a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/capture/capture.mjs +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/capture/capture.mjs @@ -274,7 +274,10 @@ async function run() { sleep(holdSeconds * 1000); } else { const holdMs = beat.holdMs ?? 900; - pushNote(settled + 0.15, settled + holdMs / 1000 + 0.6); + // End AT the hold boundary: the next beat may scroll immediately, + // and the anchor is only valid for this scroll position. The + // renderer's 0.35s fade-out completes inside the hold. + pushNote(settled + 0.15, settled + holdMs / 1000); sleep(holdMs); } continue;