From 74485b1f89cd1695ecd0dedc49e58301b256652f Mon Sep 17 00:00:00 2001 From: pannous Date: Sun, 15 Mar 2026 14:15:57 +0100 Subject: [PATCH 1/2] feat: add timeline track and clip creation functionality Implement the ability to create new tracks and clips directly in the timeline editor, addressing issue #10: 1. createTrack(category) - creates a new track with the specified segment category (VIDEO, IMAGE, DIALOGUE, etc.) 2. createClip({track, startTimeInMs, ...}) - creates a new clip/segment on a given track at the cursor position 3. moveSegmentToTrack(segment, targetTrack) - moves a segment between tracks with category compatibility validation and collision detection 4. TimelineToolbar UI component with category selector, track selector, and buttons for creating tracks and clips Resolves #10 --- .../core/timeline/TimelineToolbar.tsx | 124 ++++++++++++++ .../src/components/core/timeline/index.tsx | 14 +- packages/timeline/src/hooks/useTimeline.ts | 157 +++++++++++++++++- packages/timeline/src/types/timeline.ts | 25 ++- 4 files changed, 311 insertions(+), 9 deletions(-) create mode 100644 packages/app/src/components/core/timeline/TimelineToolbar.tsx diff --git a/packages/app/src/components/core/timeline/TimelineToolbar.tsx b/packages/app/src/components/core/timeline/TimelineToolbar.tsx new file mode 100644 index 00000000..0c4f6f37 --- /dev/null +++ b/packages/app/src/components/core/timeline/TimelineToolbar.tsx @@ -0,0 +1,124 @@ +'use client' + +import { useState } from 'react' +import { useTimeline } from '@aitube/timeline' +import { ClapSegmentCategory } from '@aitube/clap' +import { Plus, ListPlus } from 'lucide-react' + +import { cn } from '@/lib/utils' +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from '@/components/ui/select' + +const TRACK_CATEGORIES: { value: ClapSegmentCategory; label: string }[] = [ + { value: ClapSegmentCategory.VIDEO, label: 'Video' }, + { value: ClapSegmentCategory.IMAGE, label: 'Image' }, + { value: ClapSegmentCategory.DIALOGUE, label: 'Dialogue' }, + { value: ClapSegmentCategory.MUSIC, label: 'Music' }, + { value: ClapSegmentCategory.SOUND, label: 'Sound' }, + { value: ClapSegmentCategory.CAMERA, label: 'Camera' }, + { value: ClapSegmentCategory.ACTION, label: 'Action' }, + { value: ClapSegmentCategory.CHARACTER, label: 'Character' }, + { value: ClapSegmentCategory.LOCATION, label: 'Location' }, + { value: ClapSegmentCategory.LIGHTING, label: 'Lighting' }, + { value: ClapSegmentCategory.STYLE, label: 'Style' }, +] + +export function TimelineToolbar() { + const [selectedCategory, setSelectedCategory] = useState( + ClapSegmentCategory.VIDEO + ) + const [selectedTrack, setSelectedTrack] = useState('') + + const tracks = useTimeline((s) => s.tracks) + const createTrack = useTimeline((s) => s.createTrack) + const createClip = useTimeline((s) => s.createClip) + const cursorTimestampAtInMs = useTimeline((s) => s.cursorTimestampAtInMs) + + const handleCreateTrack = () => { + const newTrackId = createTrack(selectedCategory) + setSelectedTrack(String(newTrackId)) + } + + const handleCreateClip = async () => { + const trackNumber = selectedTrack ? parseInt(selectedTrack, 10) : -1 + if (trackNumber < 0 || !tracks[trackNumber]) return + + await createClip({ + track: trackNumber, + startTimeInMs: cursorTimestampAtInMs || 0, + category: selectedCategory, + }) + } + + return ( +
+ + + + + + + +
+ ) +} diff --git a/packages/app/src/components/core/timeline/index.tsx b/packages/app/src/components/core/timeline/index.tsx index 2a15af9c..64c205a4 100644 --- a/packages/app/src/components/core/timeline/index.tsx +++ b/packages/app/src/components/core/timeline/index.tsx @@ -4,6 +4,7 @@ import { ClapTimeline, useTimeline, SegmentResolver } from '@aitube/timeline' import { useMonitor } from '@/services/monitor/useMonitor' import { useResolver } from '@/services/resolver/useResolver' import { useUI } from '@/services/ui' +import { TimelineToolbar } from './TimelineToolbar' export function Timeline( { @@ -56,13 +57,12 @@ export function Timeline( togglePlayback, ]) - if (className) { - return ( -
+ return ( +
+ +
- ) - } - - return +
+ ) } diff --git a/packages/timeline/src/hooks/useTimeline.ts b/packages/timeline/src/hooks/useTimeline.ts index c5276f4a..bb58ed2e 100644 --- a/packages/timeline/src/hooks/useTimeline.ts +++ b/packages/timeline/src/hooks/useTimeline.ts @@ -1,7 +1,7 @@ import { create } from "zustand" import * as THREE from "three" import type { ThreeEvent } from "@react-three/fiber" -import { ClapProject, ClapSegment, ClapSegmentCategory, isValidNumber, newClap, serializeClap, ClapTracks, ClapEntity, ClapMeta } from "@aitube/clap" +import { ClapProject, ClapSegment, ClapSegmentCategory, ClapOutputType, isValidNumber, newClap, newSegment, serializeClap, ClapTracks, ClapEntity, ClapMeta } from "@aitube/clap" import { TimelineSegment, SegmentEditionStatus, SegmentVisibility, TimelineStore, SegmentArea, SegmentPointerEvent, SegmentEventCallbackHandler, Invalidate } from "@/types/timeline" import { getDefaultProjectState, getDefaultState } from "@/utils/getDefaultState" @@ -1206,6 +1206,161 @@ export const useTimeline = create((set, get) => ({ }) }) }, + createTrack: (category: ClapSegmentCategory): number => { + const { + width, + height, + tracks, + cellWidth, + defaultSegmentDurationInSteps, + durationInMsPerStep, + durationInMs, + defaultPreviewHeight, + defaultCellHeight, + } = get() + + const isPreview = + category === ClapSegmentCategory.IMAGE || + category === ClapSegmentCategory.VIDEO + + const newTrackId = tracks.length + + const updatedTracks = [ + ...tracks, + { + id: newTrackId, + name: `${category}`, + isPreview, + height: isPreview ? defaultPreviewHeight : defaultCellHeight, + hue: 0, + occupied: false, + visible: true, + }, + ] + + set({ + ...computeContentSizeMetrics({ + width, + height, + tracks: updatedTracks, + cellWidth, + defaultSegmentDurationInSteps, + durationInMsPerStep, + durationInMs, + }), + }) + + return newTrackId + }, + + createClip: async ({ + track, + startTimeInMs, + endTimeInMs, + category, + prompt = '', + }: { + track: number + startTimeInMs: number + endTimeInMs?: number + category?: ClapSegmentCategory + prompt?: string + }): Promise => { + const { + tracks, + durationInMsPerStep, + defaultSegmentDurationInSteps, + addSegment, + } = get() + + const trackInfo = tracks[track] + const trackCategory = category || ( + trackInfo + ? ClapSegmentCategory[trackInfo.name as keyof typeof ClapSegmentCategory] || ClapSegmentCategory.GENERIC + : ClapSegmentCategory.GENERIC + ) + + const defaultDurationInMs = defaultSegmentDurationInSteps * durationInMsPerStep + const segmentEndTimeInMs = endTimeInMs || (startTimeInMs + defaultDurationInMs) + + const outputType = + trackCategory === ClapSegmentCategory.VIDEO ? ClapOutputType.VIDEO + : trackCategory === ClapSegmentCategory.IMAGE ? ClapOutputType.IMAGE + : trackCategory === ClapSegmentCategory.DIALOGUE ? ClapOutputType.AUDIO + : trackCategory === ClapSegmentCategory.MUSIC ? ClapOutputType.AUDIO + : trackCategory === ClapSegmentCategory.SOUND ? ClapOutputType.AUDIO + : ClapOutputType.TEXT + + const clapSegment = newSegment({ + track, + startTimeInMs, + endTimeInMs: segmentEndTimeInMs, + category: trackCategory, + prompt, + outputType, + }) + + const segment = await clapSegmentToTimelineSegment(clapSegment) + + await addSegment({ segment, startTimeInMs, track }) + + return segment + }, + + moveSegmentToTrack: (segment: TimelineSegment, targetTrack: number): boolean => { + const { + tracks, + segments, + invalidate, + allSegmentsChanged: prevAllChanged, + atLeastOneSegmentChanged: prevOneChanged, + } = get() + + const targetTrackInfo = tracks[targetTrack] + if (!targetTrackInfo) { return false } + + // Allow moves to empty/misc tracks or same-category tracks + const targetName = targetTrackInfo.name + const isCompatible = + !targetTrackInfo.occupied || + targetName === '(empty)' || + targetName === '(misc)' || + targetName === segment.category + + if (!isCompatible) { return false } + + // Check for time-overlap collisions on the target track + const hasCollision = segments.some( + (s) => + s.id !== segment.id && + s.track === targetTrack && + !(s.endTimeInMs <= segment.startTimeInMs || s.startTimeInMs >= segment.endTimeInMs) + ) + + if (hasCollision) { return false } + + segment.track = targetTrack + + // Update track info if it was empty + if (!targetTrackInfo.occupied) { + const isPreview = + segment.category === ClapSegmentCategory.IMAGE || + segment.category === ClapSegmentCategory.VIDEO + targetTrackInfo.name = `${segment.category}` + targetTrackInfo.occupied = true + targetTrackInfo.isPreview = isPreview + } + + set({ + allSegmentsChanged: prevAllChanged + 1, + atLeastOneSegmentChanged: prevOneChanged + 1, + }) + + invalidate() + + return true + }, + deleteSegments: (ids: string[]): void => { const { segments: previousSegments, diff --git a/packages/timeline/src/types/timeline.ts b/packages/timeline/src/types/timeline.ts index 71f9a4f7..e6d1e057 100644 --- a/packages/timeline/src/types/timeline.ts +++ b/packages/timeline/src/types/timeline.ts @@ -1,6 +1,6 @@ import * as THREE from "three" import type { ThreeEvent } from "@react-three/fiber" -import { ClapEntity, ClapImageRatio, ClapMeta, ClapProject, ClapScene, ClapSegment, ClapTracks } from "@aitube/clap" +import { ClapEntity, ClapImageRatio, ClapMeta, ClapProject, ClapScene, ClapSegment, ClapSegmentCategory, ClapTracks } from "@aitube/clap" import { ClapSegmentColorScheme, ClapTimelineTheme } from "./theme" import { TimelineControlsImpl } from "@/components/controls/types" @@ -397,6 +397,29 @@ export type TimelineStoreModifiers = { */ deleteSegments: (ids: string[]) => void + /** + * Create a new track with a given category + * @returns the track number of the newly created track + */ + createTrack: (category: ClapSegmentCategory) => number + + /** + * Create a new clip/segment on a track at a given time position + */ + createClip: (params: { + track: number + startTimeInMs: number + endTimeInMs?: number + category?: ClapSegmentCategory + prompt?: string + }) => Promise + + /** + * Move a segment to a different track, validating category compatibility + * @returns true if the move succeeded + */ + moveSegmentToTrack: (segment: TimelineSegment, targetTrack: number) => boolean + addEntities: (entities: ClapEntity[]) => Promise updateEntities: (entities: ClapEntity[]) => Promise deleteEntities: (entities: (ClapEntity|string)[]) => Promise From 172dd5545cc71153320385b28de4ee538eda92f9 Mon Sep 17 00:00:00 2001 From: pannous Date: Mon, 16 Mar 2026 10:57:11 +0100 Subject: [PATCH 2/2] fix: address Copilot review suggestions on PR #136 - createTrack: mark new typed tracks as occupied:true so they're constrained to the chosen category - createClip: add collision check before addSegment, fall back to findFreeTrack on overlap - moveSegmentToTrack: use immutable updates for segments/tracks arrays and recompute metrics so React re-renders - Timeline className: use cn() to always apply base flex layout classes and merge optional className --- .../src/components/core/timeline/index.tsx | 3 +- packages/timeline/src/hooks/useTimeline.ts | 63 +++++++++++++++---- 2 files changed, 52 insertions(+), 14 deletions(-) diff --git a/packages/app/src/components/core/timeline/index.tsx b/packages/app/src/components/core/timeline/index.tsx index 64c205a4..719b8bae 100644 --- a/packages/app/src/components/core/timeline/index.tsx +++ b/packages/app/src/components/core/timeline/index.tsx @@ -4,6 +4,7 @@ import { ClapTimeline, useTimeline, SegmentResolver } from '@aitube/timeline' import { useMonitor } from '@/services/monitor/useMonitor' import { useResolver } from '@/services/resolver/useResolver' import { useUI } from '@/services/ui' +import { cn } from '@/lib/utils' import { TimelineToolbar } from './TimelineToolbar' export function Timeline( @@ -58,7 +59,7 @@ export function Timeline( ]) return ( -
+
diff --git a/packages/timeline/src/hooks/useTimeline.ts b/packages/timeline/src/hooks/useTimeline.ts index bb58ed2e..c467bf74 100644 --- a/packages/timeline/src/hooks/useTimeline.ts +++ b/packages/timeline/src/hooks/useTimeline.ts @@ -1233,7 +1233,7 @@ export const useTimeline = create((set, get) => ({ isPreview, height: isPreview ? defaultPreviewHeight : defaultCellHeight, hue: 0, - occupied: false, + occupied: true, visible: true, }, ] @@ -1291,8 +1291,20 @@ export const useTimeline = create((set, get) => ({ : trackCategory === ClapSegmentCategory.SOUND ? ClapOutputType.AUDIO : ClapOutputType.TEXT + // Check for time-overlap collisions on the requested track + const { segments, findFreeTrack: findFree } = get() + const hasCollision = segments.some( + (s) => + s.track === track && + !(s.endTimeInMs <= startTimeInMs || s.startTimeInMs >= segmentEndTimeInMs) + ) + + const resolvedTrack = hasCollision + ? findFree({ startTimeInMs, endTimeInMs: segmentEndTimeInMs }) + : track + const clapSegment = newSegment({ - track, + track: resolvedTrack, startTimeInMs, endTimeInMs: segmentEndTimeInMs, category: trackCategory, @@ -1302,15 +1314,21 @@ export const useTimeline = create((set, get) => ({ const segment = await clapSegmentToTimelineSegment(clapSegment) - await addSegment({ segment, startTimeInMs, track }) + await addSegment({ segment, startTimeInMs, track: resolvedTrack }) return segment }, moveSegmentToTrack: (segment: TimelineSegment, targetTrack: number): boolean => { const { + width, + height, tracks, segments, + cellWidth, + defaultSegmentDurationInSteps, + durationInMsPerStep, + durationInMs, invalidate, allSegmentsChanged: prevAllChanged, atLeastOneSegmentChanged: prevOneChanged, @@ -1339,21 +1357,40 @@ export const useTimeline = create((set, get) => ({ if (hasCollision) { return false } - segment.track = targetTrack + // Immutable update: create new segment with updated track + const updatedSegments = segments.map((s) => + s.id === segment.id ? { ...s, track: targetTrack } : s + ) - // Update track info if it was empty - if (!targetTrackInfo.occupied) { - const isPreview = - segment.category === ClapSegmentCategory.IMAGE || - segment.category === ClapSegmentCategory.VIDEO - targetTrackInfo.name = `${segment.category}` - targetTrackInfo.occupied = true - targetTrackInfo.isPreview = isPreview - } + // Immutable update: create new tracks array if target was empty + const updatedTracks = !targetTrackInfo.occupied + ? tracks.map((t, i) => { + if (i !== targetTrack) { return t } + const isPreview = + segment.category === ClapSegmentCategory.IMAGE || + segment.category === ClapSegmentCategory.VIDEO + return { + ...t, + name: `${segment.category}`, + occupied: true, + isPreview, + } + }) + : [...tracks] set({ + segments: updatedSegments, allSegmentsChanged: prevAllChanged + 1, atLeastOneSegmentChanged: prevOneChanged + 1, + ...computeContentSizeMetrics({ + width, + height, + tracks: updatedTracks, + cellWidth, + defaultSegmentDurationInSteps, + durationInMsPerStep, + durationInMs, + }), }) invalidate()