diff --git a/packages/cloud-agents/src/server/workflows/__tests__/featureDemoSkill.test.ts b/packages/cloud-agents/src/server/workflows/__tests__/featureDemoSkill.test.ts index b80485b94..759ce3441 100644 --- a/packages/cloud-agents/src/server/workflows/__tests__/featureDemoSkill.test.ts +++ b/packages/cloud-agents/src/server/workflows/__tests__/featureDemoSkill.test.ts @@ -132,6 +132,31 @@ describe('feature-demo skill', () => { expect(captureRunner).toContain('pushCursorMove(moveStart, moveEnd, c);'); }); + it('anchors annotations to capture-measured element rects', () => { + const captureRunner = fs.readFileSync( + path.join(skillDirPath, 'capture/capture.mjs'), + 'utf8', + ); + const annotationsComponent = fs.readFileSync( + path.join(skillDirPath, 'render/src/Annotations.tsx'), + 'utf8', + ); + + // Capture measures the anchor (tight content rect, resolved after the + // beat's scroll settles) and scopes it to the beat's caption window; + // the renderer only draws at capture-emitted rects. + expect(captureRunner).toContain('annotations: []'); + expect(captureRunner).toContain('function tightRect('); + expect(captureRunner).toContain('selectNodeContents'); + expect(captureRunner).toContain("beat.note.style ?? 'spotlight'"); + // The layer is clipped to the window so a spotlight dim can never touch + // the backdrop or the caption band. + expect(annotationsComponent).toContain("overflow: 'hidden'"); + // Documented as a rare, single-per-beat clarity device. + expect(skillContent).toContain('ONE `"note"`'); + expect(skillContent).toContain('not decoration'); + }); + it.each(['focus', 'reset'])('rejects the unsupported %s action', (action) => { const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'feature-demo-')); const scriptPath = path.join(tempDir, 'demo-script.json'); diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md index 501796f47..938d62114 100644 --- a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md @@ -57,6 +57,7 @@ this pipeline is ever committed to the repository. Write `/tmp/feature-demo/demo-script.json`: `{ "url": string, "viewport": { "w": 1280, "h": 800 }, "beats": [...] }`. Beat actions: - `{ "a": "show", "sel": css, "caption": "..." }` — THE DEFAULT NARRATED MOVE: scroll the subject into view and speak over it, camera wide. No cursor. +- A `show` beat may carry ONE `"note"` — `{ "sel": css?, "text": "..."?, "style": "spotlight"|"box"|"callout"? }` — an attention cue anchored to a real element for exactly that beat's caption window. Default `spotlight` dims the rest of the page and leaves the target bright, with `text` as a small label on the dimmed area; use it when the narration names one thing the viewer should find (`note.sel` may point at a more specific element than the beat scrolls to, e.g. the command block under a heading). `box`/`callout` draw an accent outline instead — busier on dense pages, so prefer spotlight. At most one note per beat, and most beats want none: it is a clarity device, not decoration. - `{ "a": "wait", "ms": n }` / `{ "a": "hold", "ms": n }` — let the page settle / linger. - `{ "a": "scrollTo", "sel": css, "ms": n }` — plain scroll with no narration attached. - `{ "a": "click", "sel": css, "holdMs": ~300 }` — real click with ripple. diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/capture/capture.mjs b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/capture/capture.mjs index f70066690..b5fea086c 100644 --- a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/capture/capture.mjs +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/capture/capture.mjs @@ -118,6 +118,39 @@ const centerNorm = (r) => ({ y: (r.y + r.h / 2) / VIEWPORT.h, }); +// Full normalized bounding box — the anchor for annotations. Resolved AFTER +// the beat's scroll settles, because rects are viewport-relative: an anchor +// is only valid for the scroll position it was measured at, which is why +// annotations live strictly inside their beat's window. +const boxNorm = (r) => ({ + x: r.x / VIEWPORT.w, + y: r.y / VIEWPORT.h, + w: r.w / VIEWPORT.w, + h: r.h / VIEWPORT.h, +}); + +// Tight content rect for annotation anchors. Block elements (headings, +// paragraphs) report full-column boxes with dead space past the text; a +// Range over the contents hugs what the viewer actually reads. Falls back +// to the element box for empty/replaced elements. +function tightRect(sel) { + const selB64 = Buffer.from(String(sel), 'utf8').toString('base64'); + if (!/^[A-Za-z0-9+/=]*$/.test(selB64)) { + throw new Error(`unencodable selector: ${sel}`); + } + const js = + `(function(){var e=document.querySelector(atob("${selB64}"));` + + `if(!e)return null;var r=e.getBoundingClientRect();` + + `try{var g=document.createRange();g.selectNodeContents(e);` + + `var t=g.getBoundingClientRect();` + + `if(t&&t.width>1&&t.height>1)r=t;}catch(_){}` + + `return{x:r.x,y:r.y,w:r.width,h:r.height};})()`; + const out = ab('eval', js).trim(); + const r = JSON.parse(out); + if (!r) throw new Error(`element not found: ${sel}`); + return r; +} + const timeline = { video: { path: 'recording.mp4', width: VIEWPORT.w, height: VIEWPORT.h }, fps: 30, @@ -125,6 +158,7 @@ const timeline = { cursorKeys: [{ t: 0, v: { x: 0.5, y: 1.1 } }], clicks: [], captions: [], + annotations: [], // Optional declarative caption styling from the demo script (position, // accent, pill, sizeScale); the renderer merges it over preset defaults. ...(script.captionStyle ? { captionStyle: script.captionStyle } : {}), @@ -195,6 +229,23 @@ async function run() { sleep(beat.settleMs ?? 600); // scroll settles on screen const settled = now(); + // `note` rides the beat: a highlight box (plus optional label chip) + // anchored to the beat's own element, shown for the beat's caption + // window. One per beat — it is a clarity device, not a diagram. + const noteBox = beat.note + ? boxNorm(tightRect(beat.note.sel ?? beat.sel)) + : null; + const pushNote = (start, end) => { + if (!noteBox) return; + timeline.annotations.push({ + start: Math.round(start * 1000) / 1000, + end: Math.round(end * 1000) / 1000, + box: noteBox, + ...(beat.note.text ? { text: beat.note.text } : {}), + style: beat.note.style ?? 'spotlight', + }); + }; + if (beat.caption) { const lineSeconds = narration ? narration.clips[lineIndex].durationSeconds @@ -215,10 +266,19 @@ async function run() { } lineIndex += 1; + // The note breathes with the caption: appears a beat after the line + // starts (the voice names the subject, then the box lands on it). + pushNote(lineStart + 0.35, lineEnd + 0.25); + const holdSeconds = Math.max(0.5, lineEnd + LINE_GAP - now()); sleep(holdSeconds * 1000); } else { - sleep(beat.holdMs ?? 900); + const holdMs = beat.holdMs ?? 900; + // End AT the hold boundary: the next beat may scroll immediately, + // and the anchor is only valid for this scroll position. The + // renderer's 0.35s fade-out completes inside the hold. + pushNote(settled + 0.15, settled + holdMs / 1000); + sleep(holdMs); } continue; } diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/README.md b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/README.md index c9d1e58ac..0ac94e239 100644 --- a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/README.md +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/README.md @@ -10,13 +10,22 @@ render those; adaptations change how they look, not what they mean. Worth preserving when adapting — each encodes a bug class found the hard way: -- **Counter-scaled cursor/ripple** (`DemoStage.tsx`): cursor and ripple are - children of the zoom-transform container with a `1/S` counter-scale, so - they stay glued to page coordinates at constant size through any zoom. -- **Edge-clamp guard** (`DemoStage.tsx`): the translate that keeps backdrop - from showing behind the window must only clamp an axis when the scaled - window exceeds the canvas on that axis; clamping a smaller-than-canvas - window (vertical presets) shoves it into a corner. +- **Counter-scaled cursor/ripple/annotations** (`DemoStage.tsx`, + `Annotations.tsx`): overlays are children of the window transform container + with a `1/S` counter-scale on their chrome, so they stay glued to page + coordinates at constant on-screen size under a preset `baseScale`. +- **Unconditional edge clamps** (`DemoStage.tsx`): the same pair of translate + bounds means "keep covering the stage" when the scaled window exceeds it + and "stay inside the stage" when it does not — the two just swap order — + so clamping to their min/max is correct in both regimes. Do not gate the + clamps on window size; the gated version let a nearly-stage-height window + drift into the caption band. +- **Window-clipped annotations** (`Annotations.tsx`): the annotation layer is + clipped to the window rect (same 16px radius as the video panel) so a + spotlight dim covers exactly the recording — never the backdrop or the + caption band — and each annotation lives strictly inside its beat's + caption window, because its anchor rect is only valid for the scroll + position it was measured at. - **Timeline-driven interpolation**: all motion eases between capture-emitted keys. Do not invent motion that is not in the timeline — it will drift from the recording. diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/Annotations.tsx b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/Annotations.tsx new file mode 100644 index 000000000..780f5df5d --- /dev/null +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/Annotations.tsx @@ -0,0 +1,143 @@ +// Annotation layer: capture-anchored attention cues drawn over the +// recording. Rendered INSIDE the window transform container (like the +// cursor and click ripple) so anchors stay glued to page coordinates, with +// chrome counter-scaled by 1/S so stroke widths and chip text keep constant +// on-screen size under a preset baseScale. +// +// The default style is `spotlight`: dim the rest of the window and leave +// the target bright. Attention comes from contrast, not from chrome drawn +// onto the page — boxes and labels over a dense real page read as stickers, +// which is why the outline styles are opt-in. +// +// Anchors are resolved at beat-settle time and are only valid for that +// scroll position, so each annotation lives strictly inside its beat's +// caption window — capture scopes the timing, the renderer only fades. + +type Box = { x: number; y: number; w: number; h: number }; + +export type Annotation = { + start: number; + end: number; + box: Box; + text?: string; + style?: 'spotlight' | 'box' | 'callout'; +}; + +const clamp = (v: number, lo: number, hi: number) => + Math.max(lo, Math.min(hi, v)); + +export const Annotations: React.FC<{ + annotations: Annotation[]; + t: number; + baseW: number; + baseH: number; + invScale: number; + accent: string; +}> = ({ annotations, t, baseW, baseH, invScale, accent }) => { + return ( + // Clip to the window rect (same radius as the video panel) so the + // spotlight dim covers exactly the recording, never the backdrop. +
+ {annotations.map((a, i) => { + const fadeIn = clamp((t - a.start) / 0.35, 0, 1); + const fadeOut = clamp((a.end - t) / 0.35, 0, 1); + const opacity = Math.min(fadeIn, fadeOut); + if (opacity <= 0) return null; + + // Breathing room around the measured content rect. + const padX = 14; + const padY = 10; + const left = a.box.x * baseW - padX; + const top = a.box.y * baseH - padY; + const width = a.box.w * baseW + 2 * padX; + const height = a.box.h * baseH + 2 * padY; + + const spotlight = a.style !== 'box' && a.style !== 'callout'; + const showChip = a.text && a.style !== 'box'; + // Chip sits on the dimmed area above the cutout, left-aligned with + // it (or below when the target is near the top edge). Its rendered + // width is estimated from the label (nowrap, so width tracks text) + // to keep the right edge inside the window. + const chipAbove = top > 72 * invScale; + const chipW = ((a.text?.length ?? 0) * 17 * 0.56 + 28) * invScale; + const chipLeft = clamp(left, 12, Math.max(12, baseW - 12 - chipW)); + + return ( +
+ {spotlight ? ( + // The cutout: a transparent rounded rect whose enormous + // box-shadow dims everything else in the (clipped) window. +
+ ) : ( +
+ )} + {showChip ? ( +
+
+ {a.text} +
+
+ ) : null} +
+ ); + })} +
+ ); +}; diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/DemoStage.tsx b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/DemoStage.tsx index 2c5794df0..509876489 100644 --- a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/DemoStage.tsx +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/DemoStage.tsx @@ -1,6 +1,7 @@ import { OffthreadVideo, staticFile, useCurrentFrame } from 'remotion'; import timeline from '../props/timeline.json'; import narration from '../props/narration.json'; +import { Annotations, type Annotation } from './Annotations'; import type { CaptionStyle } from './presets'; type WordTiming = { text: string; start: number; end: number }; @@ -262,6 +263,22 @@ export const DemoStage: React.FC<{
) : null} + + {/* Annotation layer: same page-coordinate space as the cursor and + ripple, chrome counter-scaled so it holds constant on-screen + size under a preset baseScale. Older timelines have no + annotations key and render unchanged. */} +