Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 16 additions & 12 deletions client/src/components/cos/MiniCharacterCoSAvatar.jsx
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import CoSAvatarOrbitControls from './CoSAvatarOrbitControls';
import CoSBackgroundCamera from './CoSBackgroundCamera';
import CoSCanvasGuard from './CoSCanvasGuard';
import useClonedGltf, { GltfPrimitive } from '../../hooks/useClonedGltf';
import { resolvePlaybackClip } from '../../hooks/useAvatarCapabilities';
import { fitModelToHeight } from '../../utils/modelFit';

// Kenney Mini Characters (CC0) ship 32 named clips. We map the CoS agent
Expand Down Expand Up @@ -39,7 +40,7 @@ function buildModelUrl(variant) {
return variant ? `/api/avatar/model.glb?variant=${encodeURIComponent(variant)}` : '/api/avatar/model.glb';
}

function MiniCharacter({ state, speaking, variant }) {
function MiniCharacter({ state, speaking, variant, coverage = null }) {
const url = useMemo(() => buildModelUrl(variant), [variant]);
const group = useRef();
const { scene, actions, names } = useClonedGltf(url);
Expand All @@ -52,13 +53,14 @@ function MiniCharacter({ state, speaking, variant }) {
fitModelToHeight(scene, { targetHeight: TARGET_HEIGHT, feetOnGround: true, yOffset: GROUND_Y });
}, [scene]);

// Resolve the active clip for this state, falling back gracefully.
// Resolve the active clip for this state, falling back gracefully. With a
// coverage report (a rigged record, #5894) the covered clip wins when the
// GLB carries it, else playback degrades to a clip the character actually
// has — an uncovered state never freezes the frame or pretends coverage.
const cfg = STATE_CLIP_MAP[state] || FALLBACK;
const clipName = useMemo(() => {
if (names.includes(cfg.clip)) return cfg.clip;
if (names.includes('idle')) return 'idle';
return names[0];
}, [names, cfg.clip]);
const clipName = useMemo(() => (
resolvePlaybackClip(names, { state, coverage, fallbacks: [cfg.clip, 'idle'] })
), [names, state, coverage, cfg.clip]);

// Crossfade between clips on state change.
const prevClip = useRef(null);
Expand Down Expand Up @@ -107,14 +109,14 @@ function StageLighting({ color }) {
);
}

function Scene({ state, speaking, background, variant }) {
function Scene({ state, speaking, background, variant, coverage = null }) {
const stateConfig = AGENT_STATES[state] || AGENT_STATES.sleeping;
const color = stateConfig.color;
return (
<>
<CoSBackgroundCamera enabled={background} z={3.6} />
<StageLighting color={color} />
<MiniCharacter state={state} speaking={speaking} variant={variant} />
<MiniCharacter state={state} speaking={speaking} variant={variant} coverage={coverage} />
<CoSAvatarOrbitControls />
</>
);
Expand Down Expand Up @@ -148,8 +150,10 @@ function LoadingPlaceholder({ background = false }) {
);
}

// `variant` is wired by the per-character style wrappers (MiniCharMaleC, etc.).
export default function MiniCharacterCoSAvatar({ state, speaking, background = false, variant = 'mini-male-c' }) {
// `variant` is wired by the per-character style wrappers (MiniCharMaleC, etc.)
// or a `rigged-<modelId>` spelling for an animated record (ChiefOfStaff passes
// that record's coverage so playback falls back to a present clip).
export default function MiniCharacterCoSAvatar({ state, speaking, background = false, variant = 'mini-male-c', coverage = null }) {
const [modelPresent, setModelPresent] = useState(null);
const url = useMemo(() => buildModelUrl(variant), [variant]);

Expand Down Expand Up @@ -177,7 +181,7 @@ export default function MiniCharacterCoSAvatar({ state, speaking, background = f
gl={{ alpha: true, antialias: true }}
>
<Suspense fallback={null}>
<Scene state={state} speaking={speaking} background={background} variant={variant} />
<Scene state={state} speaking={speaking} background={background} variant={variant} coverage={coverage} />
</Suspense>
</Canvas>
</CoSCanvasGuard>
Expand Down
11 changes: 11 additions & 0 deletions client/src/components/cos/MiniCharacterCoSAvatar.test.jsx
Original file line number Diff line number Diff line change
Expand Up @@ -48,4 +48,15 @@ describe('MiniCharacterCoSAvatar', () => {
'/api/avatar/model.glb?variant=mini-female-d',
));
});

// A rigged record probes through the same variant namespace — the avatar
// route resolves `rigged-<modelId>` to the record's animated GLB, so the
// stage needs no special case for record-backed characters.
it('keys the canvas guard on the rigged variant url for an animated record', async () => {
render(<MiniCharacterCoSAvatar variant="rigged-image3d-1" coverage={null} />);
await waitFor(() => expect(screen.getByTestId('canvas-guard')).toHaveAttribute(
'data-reset-key',
'/api/avatar/model.glb?variant=rigged-image3d-1',
));
});
});
41 changes: 38 additions & 3 deletions client/src/components/cos/tabs/ConfigTab.jsx
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { useCallback, useEffect, useState } from 'react';
import { useCallback, useEffect, useMemo, useState } from 'react';
import { Link } from 'react-router';
import {
Activity,
Expand Down Expand Up @@ -32,6 +32,7 @@ import {
} from '../constants';
import ProviderModelSelector from '../../ProviderModelSelector';
import useProviderModels from '../../../hooks/useProviderModels';
import { coverageSummary, isRiggedAvatarStyle, riggedRecordForStyle } from '../../../hooks/useAvatarCapabilities';
import { timeAgo } from '../../../utils/formatters';

const DOMAIN_MODE_COLORS = {
Expand Down Expand Up @@ -222,7 +223,7 @@ function PersistentMindStatus({ mind, loaded, error }) {
);
}

export default function ConfigTab({ config, onUpdate, onEvaluate, avatarStyle }) {
export default function ConfigTab({ config, onUpdate, onEvaluate, avatarStyle, riggedAvatars = [] }) {
const {
providers,
availableModels,
Expand Down Expand Up @@ -273,6 +274,37 @@ export default function ConfigTab({ config, onUpdate, onEvaluate, avatarStyle })
useEffect(() => { void refreshBudgetUsage(); }, [refreshBudgetUsage]);
useAutoRefetch(refreshMindStatus, 15_000, { pollOnly: true });

// Built-in styles plus the install's verified animated records (#5894). A
// record entry carries its state coverage in the label, so what the
// character can and cannot do is visible BEFORE it is picked.
const avatarOptions = useMemo(() => ([
...Object.entries(AVATAR_STYLE_LABELS).map(([value, label]) => ({ value, label })),
...(Array.isArray(riggedAvatars) ? riggedAvatars : [])
.filter((record) => record?.variant)
.map((record) => ({
value: record.variant,
label: `${record.name} (rigged 3D) — ${coverageSummary(record.coverage)}`,
})),
]), [riggedAvatars]);

const avatarLabel = (style) => {
if (AVATAR_STYLE_LABELS[style]) return AVATAR_STYLE_LABELS[style];
const record = riggedRecordForStyle(riggedAvatars, style);
return record ? `${record.name} (rigged 3D)` : style;
};

// Honest coverage note for the staged value: which states the character
// covers, what the rest fall back to — or a warning when the record the
// saved style points at is gone.
const stagedRiggedNote = useMemo(() => {
if (!isRiggedAvatarStyle(formData.avatarStyle)) return null;
const record = riggedRecordForStyle(riggedAvatars, formData.avatarStyle);
if (!record) return 'That animated record is no longer available — pick another avatar.';
const covered = record.coverage?.coveredStates || [];
const fallback = record.clip ? `Other states play ${record.clip}.` : '';
return `${coverageSummary(record.coverage)}. Covered: ${covered.join(', ') || 'none'}. ${fallback}`.trim();
}, [formData.avatarStyle, riggedAvatars]);

const handleCancel = () => {
setFormData(getDefaultFormData(config, avatarStyle));
setEditing(false);
Expand Down Expand Up @@ -434,9 +466,12 @@ export default function ConfigTab({ config, onUpdate, onEvaluate, avatarStyle })
<SectionHeading icon={Palette} title="Appearance" description="Set the default avatar and whether active work may choose a matching style." />
</div>
<div className="grid gap-3 sm:grid-cols-2">
<ConfigRow label="Default avatar" description="Visual style used by the CoS panel." value={AVATAR_STYLE_LABELS[formData.avatarStyle] || formData.avatarStyle} editing={editing} type="select" inputValue={formData.avatarStyle} options={Object.entries(AVATAR_STYLE_LABELS).map(([value, label]) => ({ value, label }))} onChange={(value) => setFormData((current) => ({ ...current, avatarStyle: value }))} />
<ConfigRow label="Default avatar" description="Visual style used by the CoS panel." value={avatarLabel(formData.avatarStyle)} editing={editing} type="select" inputValue={formData.avatarStyle} options={avatarOptions} onChange={(value) => setFormData((current) => ({ ...current, avatarStyle: value }))} />
<ConfigRow label="Dynamic avatar" description="Switch style based on task type, provider, or priority." value={formData.dynamicAvatar ? 'Enabled' : 'Disabled'} editing={editing} type="checkbox" inputValue={formData.dynamicAvatar} onChange={(value) => setFormData((current) => ({ ...current, dynamicAvatar: value }))} />
</div>
{stagedRiggedNote && (
<p className="text-xs leading-relaxed text-port-text-muted">{stagedRiggedNote}</p>
)}
</section>

<section className="space-y-3" aria-labelledby="embeddings-heading">
Expand Down
59 changes: 59 additions & 0 deletions client/src/components/cos/tabs/ConfigTab.test.jsx
Original file line number Diff line number Diff line change
Expand Up @@ -267,6 +267,65 @@ describe('Default Avatar Style dropdown', () => {
});
});

describe('Rigged avatar records in the Default Avatar dropdown', () => {
const riggedAvatars = [{
id: 'image3d-1',
name: 'Example Dancer',
variant: 'rigged-image3d-1',
assetUrl: '/api/avatar/model.glb?variant=rigged-image3d-1',
clip: 'Dance',
coverage: {
availableClips: ['Dance'],
coverageByState: {
thinking: { covered: false, clip: null },
ideating: { covered: true, clip: 'Dance' },
},
coveredStates: ['ideating'],
missingStates: ['thinking'],
complete: false,
},
}];

it('offers verified animated records alongside the built-in styles', async () => {
renderConfig({ config: { ...config, avatarStyle: 'svg' }, riggedAvatars });
await screen.findByText('Waiting for the next wake');

fireEvent.click(screen.getByRole('button', { name: /Edit/i }));

const select = screen.getByRole('combobox', { name: 'Default avatar' });
const labels = [...select.options].map((option) => option.text);
expect(labels).toContain('Digital (SVG)');
expect(labels.some((label) => label.includes('Example Dancer') && label.includes('rigged 3D'))).toBe(true);
});

it('shows the coverage note when a rigged record is staged', async () => {
renderConfig({ config: { ...config, avatarStyle: 'svg' }, riggedAvatars });
await screen.findByText('Waiting for the next wake');

fireEvent.click(screen.getByRole('button', { name: /Edit/i }));

const select = screen.getByRole('combobox', { name: 'Default avatar' });
fireEvent.change(select, { target: { value: 'rigged-image3d-1' } });

expect(await screen.findByText(/Covered: ideating/)).toBeInTheDocument();
expect(screen.getByText(/Other states play Dance/)).toBeInTheDocument();

api.updateCosConfig.mockResolvedValue({ success: true });
fireEvent.click(screen.getByRole('button', { name: /Save/i }));
await waitFor(() => expect(api.updateCosConfig).toHaveBeenCalledWith(
expect.objectContaining({ avatarStyle: 'rigged-image3d-1' }),
{ silent: true },
));
});

it('warns when the saved rigged record is no longer offered', async () => {
renderConfig({ config: { ...config, avatarStyle: 'rigged-image3d-gone' }, riggedAvatars });
await screen.findByText('Waiting for the next wake');

expect(screen.getByText(/no longer available/)).toBeInTheDocument();
});
});

describe('persistent mind status', () => {
it('shows the live supervisor state and links to the full mind workspace', async () => {
renderConfig();
Expand Down
1 change: 1 addition & 0 deletions client/src/components/songbook/PracticeLogger.jsx
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ export default function PracticeLogger({ song, onLogged, className = '' }) {
type="button"
onClick={() => logPractice(rating.quality)}
disabled={logging}
aria-label={rating.label}
className="min-h-[48px] px-2.5 py-2 text-left sm:text-center rounded-lg border border-port-border text-gray-300 hover:text-white hover:border-port-accent/50 hover:bg-port-border/50 disabled:opacity-50 flex flex-col justify-center"
>
<span className="text-xs sm:text-sm font-semibold text-white">{rating.label}</span>
Expand Down
2 changes: 2 additions & 0 deletions client/src/hooks/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,8 @@ grep -i "what you want to do" client/src/hooks/README.md

| Hook | Purpose | Use when |
|---|---|---|
| `useAvatarCapabilities` | Verified animated records for the avatar selectors (`GET /avatar/rigged`) plus the honest state→clip resolvers (`resolveStateClip`, `resolvePlaybackClip`, `coverageSummary`): an uncovered CoS state falls back to a clip the character actually has, never a pretended one. | Any surface offering or playing a rigged-record avatar (CoS avatar selector, CoS playback). |
|---|---|---|
| `useMediaAnnotations` | Per-entry `own`/`others` annotations with back-compat aliases. | Showing media annotations + ownership. |
| `useMediaCompletionRefresh` | Refetch on image/video completion socket events. | A list view that needs to refresh when new media lands. |
| `useOpenClawAttachments` | File attachment handling (base64, size-capped). | OpenClaw attachment UI. |
Expand Down
1 change: 1 addition & 0 deletions client/src/hooks/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,7 @@ export * from './useStoryStepRuns.jsx';
export * from './useModelDownloadStatus.js';

// === Media (annotations, completion, attachments) ===
export * from './useAvatarCapabilities.js';
export * from './useMediaAnnotations.js';
export * from './useSpritePendingRenders.js';
export { default as useSpriteRecordCrud } from './useSpriteRecordCrud.js';
Expand Down
111 changes: 111 additions & 0 deletions client/src/hooks/useAvatarCapabilities.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
import { useCallback, useEffect, useState } from 'react';
import { getRiggedAvatars } from '../services/api';

// Avatar capabilities for rigged + animated records (#5894).
//
// A retargeted character carries whatever clips its ONE retarget produced —
// usually a single clip — while every CoS state wants its own motion. This
// module is the honesty layer between the two: the server reports per-state
// coverage (`GET /avatar/rigged`, computed by
// `server/services/rigging/clipCapabilities.js`), and these resolvers turn
// that report into a playable clip WITHOUT ever pretending an uncovered state
// is covered. A missing state deterministically falls back to a clip the
// character actually has.

/** `?variant=` namespace prefix for record-backed avatar styles. */
export const RIGGED_AVATAR_PREFIX = 'rigged-';

/** Whether an avatar-style value selects a rigged record vs a built-in style. */
export const isRiggedAvatarStyle = (style) => typeof style === 'string' && style.startsWith(RIGGED_AVATAR_PREFIX);

/** The selector entry for a style value, or null when it is not offered. */
export const riggedRecordForStyle = (records, style) => (
isRiggedAvatarStyle(style) && Array.isArray(records)
? records.find((record) => record?.variant === style) || null
: null
);

/**
* The clip a CoS state maps to under a server coverage report: the covered
* clip when the state is covered, else the first available clip
* (deterministic — same record, same answer), else null when the character
* carries no clip at all. Never invents coverage.
* @param {object|null} coverage The `coverage` half of a `/avatar/rigged` entry.
* @param {string} state A CoS agent state.
* @returns {string|null}
*/
export function resolveStateClip(coverage, state) {
const stateClip = coverage?.coverageByState?.[state]?.clip;
if (typeof stateClip === 'string' && stateClip) return stateClip;
const available = Array.isArray(coverage?.availableClips) ? coverage.availableClips : [];
return available.find((clip) => typeof clip === 'string' && clip) || null;
}

/**
* The clip to actually PLAY from a loaded GLB's roster. The coverage answer
* wins when the GLB still carries that clip (the record may have been
* re-retargeted since the selector read it, so presence is re-checked —
* never trusted blindly); then the caller's ordered fallbacks; then the
* roster's first clip, so an uncovered state degrades to real motion instead
* of a frozen frame. Null when the GLB carries nothing playable.
* @param {string[]} names Clip names on the loaded GLB.
* @param {{state?: string|null, coverage?: object|null, fallbacks?: string[]}} opts
* @returns {string|null}
*/
export function resolvePlaybackClip(names, { state = null, coverage = null, fallbacks = [] } = {}) {
const roster = Array.isArray(names) ? names.filter((name) => typeof name === 'string' && name) : [];
const candidates = [
...(state && coverage ? [resolveStateClip(coverage, state)] : []),
...(Array.isArray(fallbacks) ? fallbacks : []),
];
return candidates.find((clip) => clip && roster.includes(clip)) || roster[0] || null;
}

/**
* One-line honest summary of a coverage report for selector copy.
* @param {object|null} coverage
* @returns {string}
*/
export function coverageSummary(coverage) {
const states = coverage?.coverageByState ? Object.keys(coverage.coverageByState) : [];
const covered = Array.isArray(coverage?.coveredStates) ? coverage.coveredStates.length : 0;
if (states.length === 0) return 'No animation clips';
if (covered >= states.length) return `Covers all ${states.length} CoS states`;
if (covered > 0) return `Covers ${covered} of ${states.length} CoS states`;
const fallback = resolveStateClip(coverage);
return fallback ? `No covered CoS state — plays ${fallback} throughout` : 'No animation clips';
}

/**
* The install's verified animated records for the avatar selectors. Fetches
* once on mount; `refresh` re-reads (e.g. after a retarget completes
* elsewhere). Failures resolve to an empty list with `error` set — the
* selectors render their built-in styles regardless, so a rigging-lane outage
* must never take down the CoS config screen.
* @returns {{records: object[], loading: boolean, error: Error|null, refresh: Function}}
*/
export function useAvatarCapabilities() {
const [records, setRecords] = useState([]);
const [loading, setLoading] = useState(true);
const [error, setError] = useState(null);

const refresh = useCallback(async () => {
setLoading(true);
setError(null);
try {
const data = await getRiggedAvatars({ silent: true });
setRecords(Array.isArray(data?.records) ? data.records : []);
} catch (err) {
setRecords([]);
setError(err);
} finally {
setLoading(false);
}
}, []);

useEffect(() => {
refresh();
}, [refresh]);

return { records, loading, error, refresh };
}
Loading