diff --git a/packages/app/src/components/core/timeline/TimelineToolbar.tsx b/packages/app/src/components/core/timeline/TimelineToolbar.tsx new file mode 100644 index 00000000..0c4f6f37 --- /dev/null +++ b/packages/app/src/components/core/timeline/TimelineToolbar.tsx @@ -0,0 +1,124 @@ +'use client' + +import { useState } from 'react' +import { useTimeline } from '@aitube/timeline' +import { ClapSegmentCategory } from '@aitube/clap' +import { Plus, ListPlus } from 'lucide-react' + +import { cn } from '@/lib/utils' +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from '@/components/ui/select' + +const TRACK_CATEGORIES: { value: ClapSegmentCategory; label: string }[] = [ + { value: ClapSegmentCategory.VIDEO, label: 'Video' }, + { value: ClapSegmentCategory.IMAGE, label: 'Image' }, + { value: ClapSegmentCategory.DIALOGUE, label: 'Dialogue' }, + { value: ClapSegmentCategory.MUSIC, label: 'Music' }, + { value: ClapSegmentCategory.SOUND, label: 'Sound' }, + { value: ClapSegmentCategory.CAMERA, label: 'Camera' }, + { value: ClapSegmentCategory.ACTION, label: 'Action' }, + { value: ClapSegmentCategory.CHARACTER, label: 'Character' }, + { value: ClapSegmentCategory.LOCATION, label: 'Location' }, + { value: ClapSegmentCategory.LIGHTING, label: 'Lighting' }, + { value: ClapSegmentCategory.STYLE, label: 'Style' }, +] + +export function TimelineToolbar() { + const [selectedCategory, setSelectedCategory] = useState( + ClapSegmentCategory.VIDEO + ) + const [selectedTrack, setSelectedTrack] = useState('') + + const tracks = useTimeline((s) => s.tracks) + const createTrack = useTimeline((s) => s.createTrack) + const createClip = useTimeline((s) => s.createClip) + const cursorTimestampAtInMs = useTimeline((s) => s.cursorTimestampAtInMs) + + const handleCreateTrack = () => { + const newTrackId = createTrack(selectedCategory) + setSelectedTrack(String(newTrackId)) + } + + const handleCreateClip = async () => { + const trackNumber = selectedTrack ? parseInt(selectedTrack, 10) : -1 + if (trackNumber < 0 || !tracks[trackNumber]) return + + await createClip({ + track: trackNumber, + startTimeInMs: cursorTimestampAtInMs || 0, + category: selectedCategory, + }) + } + + return ( +
+ + + + + + + +
+ ) +} diff --git a/packages/app/src/components/core/timeline/index.tsx b/packages/app/src/components/core/timeline/index.tsx index 2a15af9c..719b8bae 100644 --- a/packages/app/src/components/core/timeline/index.tsx +++ b/packages/app/src/components/core/timeline/index.tsx @@ -4,6 +4,8 @@ import { ClapTimeline, useTimeline, SegmentResolver } from '@aitube/timeline' import { useMonitor } from '@/services/monitor/useMonitor' import { useResolver } from '@/services/resolver/useResolver' import { useUI } from '@/services/ui' +import { cn } from '@/lib/utils' +import { TimelineToolbar } from './TimelineToolbar' export function Timeline( { @@ -56,13 +58,12 @@ export function Timeline( togglePlayback, ]) - if (className) { - return ( -
+ return ( +
+ +
- ) - } - - return +
+ ) } diff --git a/packages/timeline/src/hooks/useTimeline.ts b/packages/timeline/src/hooks/useTimeline.ts index c5276f4a..c467bf74 100644 --- a/packages/timeline/src/hooks/useTimeline.ts +++ b/packages/timeline/src/hooks/useTimeline.ts @@ -1,7 +1,7 @@ import { create } from "zustand" import * as THREE from "three" import type { ThreeEvent } from "@react-three/fiber" -import { ClapProject, ClapSegment, ClapSegmentCategory, isValidNumber, newClap, serializeClap, ClapTracks, ClapEntity, ClapMeta } from "@aitube/clap" +import { ClapProject, ClapSegment, ClapSegmentCategory, ClapOutputType, isValidNumber, newClap, newSegment, serializeClap, ClapTracks, ClapEntity, ClapMeta } from "@aitube/clap" import { TimelineSegment, SegmentEditionStatus, SegmentVisibility, TimelineStore, SegmentArea, SegmentPointerEvent, SegmentEventCallbackHandler, Invalidate } from "@/types/timeline" import { getDefaultProjectState, getDefaultState } from "@/utils/getDefaultState" @@ -1206,6 +1206,198 @@ export const useTimeline = create((set, get) => ({ }) }) }, + createTrack: (category: ClapSegmentCategory): number => { + const { + width, + height, + tracks, + cellWidth, + defaultSegmentDurationInSteps, + durationInMsPerStep, + durationInMs, + defaultPreviewHeight, + defaultCellHeight, + } = get() + + const isPreview = + category === ClapSegmentCategory.IMAGE || + category === ClapSegmentCategory.VIDEO + + const newTrackId = tracks.length + + const updatedTracks = [ + ...tracks, + { + id: newTrackId, + name: `${category}`, + isPreview, + height: isPreview ? defaultPreviewHeight : defaultCellHeight, + hue: 0, + occupied: true, + visible: true, + }, + ] + + set({ + ...computeContentSizeMetrics({ + width, + height, + tracks: updatedTracks, + cellWidth, + defaultSegmentDurationInSteps, + durationInMsPerStep, + durationInMs, + }), + }) + + return newTrackId + }, + + createClip: async ({ + track, + startTimeInMs, + endTimeInMs, + category, + prompt = '', + }: { + track: number + startTimeInMs: number + endTimeInMs?: number + category?: ClapSegmentCategory + prompt?: string + }): Promise => { + const { + tracks, + durationInMsPerStep, + defaultSegmentDurationInSteps, + addSegment, + } = get() + + const trackInfo = tracks[track] + const trackCategory = category || ( + trackInfo + ? ClapSegmentCategory[trackInfo.name as keyof typeof ClapSegmentCategory] || ClapSegmentCategory.GENERIC + : ClapSegmentCategory.GENERIC + ) + + const defaultDurationInMs = defaultSegmentDurationInSteps * durationInMsPerStep + const segmentEndTimeInMs = endTimeInMs || (startTimeInMs + defaultDurationInMs) + + const outputType = + trackCategory === ClapSegmentCategory.VIDEO ? ClapOutputType.VIDEO + : trackCategory === ClapSegmentCategory.IMAGE ? ClapOutputType.IMAGE + : trackCategory === ClapSegmentCategory.DIALOGUE ? ClapOutputType.AUDIO + : trackCategory === ClapSegmentCategory.MUSIC ? ClapOutputType.AUDIO + : trackCategory === ClapSegmentCategory.SOUND ? ClapOutputType.AUDIO + : ClapOutputType.TEXT + + // Check for time-overlap collisions on the requested track + const { segments, findFreeTrack: findFree } = get() + const hasCollision = segments.some( + (s) => + s.track === track && + !(s.endTimeInMs <= startTimeInMs || s.startTimeInMs >= segmentEndTimeInMs) + ) + + const resolvedTrack = hasCollision + ? findFree({ startTimeInMs, endTimeInMs: segmentEndTimeInMs }) + : track + + const clapSegment = newSegment({ + track: resolvedTrack, + startTimeInMs, + endTimeInMs: segmentEndTimeInMs, + category: trackCategory, + prompt, + outputType, + }) + + const segment = await clapSegmentToTimelineSegment(clapSegment) + + await addSegment({ segment, startTimeInMs, track: resolvedTrack }) + + return segment + }, + + moveSegmentToTrack: (segment: TimelineSegment, targetTrack: number): boolean => { + const { + width, + height, + tracks, + segments, + cellWidth, + defaultSegmentDurationInSteps, + durationInMsPerStep, + durationInMs, + invalidate, + allSegmentsChanged: prevAllChanged, + atLeastOneSegmentChanged: prevOneChanged, + } = get() + + const targetTrackInfo = tracks[targetTrack] + if (!targetTrackInfo) { return false } + + // Allow moves to empty/misc tracks or same-category tracks + const targetName = targetTrackInfo.name + const isCompatible = + !targetTrackInfo.occupied || + targetName === '(empty)' || + targetName === '(misc)' || + targetName === segment.category + + if (!isCompatible) { return false } + + // Check for time-overlap collisions on the target track + const hasCollision = segments.some( + (s) => + s.id !== segment.id && + s.track === targetTrack && + !(s.endTimeInMs <= segment.startTimeInMs || s.startTimeInMs >= segment.endTimeInMs) + ) + + if (hasCollision) { return false } + + // Immutable update: create new segment with updated track + const updatedSegments = segments.map((s) => + s.id === segment.id ? { ...s, track: targetTrack } : s + ) + + // Immutable update: create new tracks array if target was empty + const updatedTracks = !targetTrackInfo.occupied + ? tracks.map((t, i) => { + if (i !== targetTrack) { return t } + const isPreview = + segment.category === ClapSegmentCategory.IMAGE || + segment.category === ClapSegmentCategory.VIDEO + return { + ...t, + name: `${segment.category}`, + occupied: true, + isPreview, + } + }) + : [...tracks] + + set({ + segments: updatedSegments, + allSegmentsChanged: prevAllChanged + 1, + atLeastOneSegmentChanged: prevOneChanged + 1, + ...computeContentSizeMetrics({ + width, + height, + tracks: updatedTracks, + cellWidth, + defaultSegmentDurationInSteps, + durationInMsPerStep, + durationInMs, + }), + }) + + invalidate() + + return true + }, + deleteSegments: (ids: string[]): void => { const { segments: previousSegments, diff --git a/packages/timeline/src/types/timeline.ts b/packages/timeline/src/types/timeline.ts index 71f9a4f7..e6d1e057 100644 --- a/packages/timeline/src/types/timeline.ts +++ b/packages/timeline/src/types/timeline.ts @@ -1,6 +1,6 @@ import * as THREE from "three" import type { ThreeEvent } from "@react-three/fiber" -import { ClapEntity, ClapImageRatio, ClapMeta, ClapProject, ClapScene, ClapSegment, ClapTracks } from "@aitube/clap" +import { ClapEntity, ClapImageRatio, ClapMeta, ClapProject, ClapScene, ClapSegment, ClapSegmentCategory, ClapTracks } from "@aitube/clap" import { ClapSegmentColorScheme, ClapTimelineTheme } from "./theme" import { TimelineControlsImpl } from "@/components/controls/types" @@ -397,6 +397,29 @@ export type TimelineStoreModifiers = { */ deleteSegments: (ids: string[]) => void + /** + * Create a new track with a given category + * @returns the track number of the newly created track + */ + createTrack: (category: ClapSegmentCategory) => number + + /** + * Create a new clip/segment on a track at a given time position + */ + createClip: (params: { + track: number + startTimeInMs: number + endTimeInMs?: number + category?: ClapSegmentCategory + prompt?: string + }) => Promise + + /** + * Move a segment to a different track, validating category compatibility + * @returns true if the move succeeded + */ + moveSegmentToTrack: (segment: TimelineSegment, targetTrack: number) => boolean + addEntities: (entities: ClapEntity[]) => Promise updateEntities: (entities: ClapEntity[]) => Promise deleteEntities: (entities: (ClapEntity|string)[]) => Promise