diff --git a/README.md b/README.md index a7895a7..de097eb 100644 --- a/README.md +++ b/README.md @@ -245,6 +245,7 @@ Creation commands also support: OAuth commands (`music cover`, `image create`, `video create`) auto-detect local paths, validate format/size, and upload to cloud storage before calling the API. OpenAPI `image create` supports local file references via base64 encoding (no size limit enforced by CLI). +OpenAPI `video create` supports local image/video/audio paths via a presigned upload URL. For Seedance, local image references automatically include width/height metadata; remote image URLs and all reference videos still need explicit metadata. ```bash # OAuth: local audio for cover (mp3, wav, flac, m4a, ogg, aac; max 20MB) @@ -256,8 +257,14 @@ listenhub image create --prompt "inspired by this" --reference ./photo.jpg # OpenAPI: local image reference (base64 encoded) listenhub openapi image create --prompt "in this style" --reference ./sketch.png --provider google -# URLs are passed through directly in both modes -listenhub openapi video create --prompt "same style" --reference-video https://example.com/clip.mp4 --input-video-duration 5 +# OpenAPI Seedance: local image references auto-populate width/height metadata +listenhub openapi video create --prompt "same style" --first-frame ./frame.png + +# Remote URLs and reference videos need dimensions to avoid server-side 32004 validation errors +listenhub openapi video create --prompt "same style" \ + --reference-video https://example.com/clip.mp4 \ + --reference-video-meta 1280x720:5:30:8000000 \ + --input-video-duration 5 ``` ## Examples @@ -288,9 +295,14 @@ listenhub openapi podcast text-stream abc123 --event script # Text-to-video listenhub openapi video create --prompt "A cat playing piano" --no-wait -j -# With first frame +# With local first frame (auto-upload + auto metadata) +listenhub openapi video create --prompt "Camera zooms out" \ + --first-frame ./frame.png + +# With remote first frame listenhub openapi video create --prompt "Camera zooms out" \ - --first-frame https://example.com/frame.png + --first-frame https://example.com/frame.png \ + --first-frame-meta 1080x1920:3600000 # Estimate credits before creating listenhub openapi video estimate --model doubao-seedance-2-pro --resolution 1080p --duration 10 diff --git a/README.zh-CN.md b/README.zh-CN.md index 3d01cb2..5d85891 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -245,6 +245,7 @@ listenhub openapi subscription -j OAuth 命令（`music cover`、`image create`、`video create`）自动检测本地路径，校验格式和大小，上传到云存储后传给 API。 OpenAPI `image create` 通过 base64 编码支持本地参考图（CLI 侧不限制文件大小）。 +OpenAPI `video create` 通过预签名 URL 支持本地图片/视频/音频路径。Seedance 的本地图片参考会自动带上宽高元数据；远程图片 URL 和所有参考视频仍需要显式 metadata。 ```bash # OAuth：本地音频用于翻唱（mp3, wav, flac, m4a, ogg, aac；最大 20MB） @@ -256,8 +257,14 @@ listenhub image create --prompt "以此为灵感" --reference ./photo.jpg # OpenAPI：本地图片参考（base64 编码） listenhub openapi image create --prompt "这种风格" --reference ./sketch.png --provider google -# URL 在两种模式下都直接透传 -listenhub openapi video create --prompt "同样风格" --reference-video https://example.com/clip.mp4 --input-video-duration 5 +# OpenAPI Seedance：本地图片参考会自动补宽高 metadata +listenhub openapi video create --prompt "同样风格" --first-frame ./frame.png + +# 远程 URL 和参考视频仍需要尺寸元数据，否则服务端会返回 32004 参数错误 +listenhub openapi video create --prompt "同样风格" \ + --reference-video https://example.com/clip.mp4 \ + --reference-video-meta 1280x720:5:30:8000000 \ + --input-video-duration 5 ``` ## 使用示例 @@ -288,9 +295,14 @@ listenhub openapi podcast text-stream abc123 --event script # 文字生成视频 listenhub openapi video create --prompt "一只猫在弹钢琴" --no-wait -j -# 指定首帧 +# 指定本地首帧（自动上传 + 自动 metadata） +listenhub openapi video create --prompt "镜头缓缓拉远" \ + --first-frame ./frame.png + +# 指定远程首帧 listenhub openapi video create --prompt "镜头缓缓拉远" \ - --first-frame https://example.com/frame.png + --first-frame https://example.com/frame.png \ + --first-frame-meta 1080x1920:3600000 # 生成前预估积分 listenhub openapi video estimate --model doubao-seedance-2-pro --resolution 1080p --duration 10 diff --git a/package.json b/package.json index 1af60ce..702fcff 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,7 @@ "prepublishOnly": "pnpm run build" }, "dependencies": { - "@marswave/listenhub-sdk": "^0.0.16", + "@marswave/listenhub-sdk": "^0.0.17", "commander": "^14.0.3", "open": "^10.0.0", "ora": "^8.0.0" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e430da5..e258eb7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -9,8 +9,8 @@ importers: .: dependencies: '@marswave/listenhub-sdk': - specifier: ^0.0.16 - version: 0.0.16 + specifier: ^0.0.17 + version: 0.0.17 commander: specifier: ^14.0.3 version: 14.0.3 @@ -57,8 +57,8 @@ packages: '@jridgewell/sourcemap-codec@1.5.5': resolution: {integrity: sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==} - '@marswave/listenhub-sdk@0.0.16': - resolution: {integrity: sha512-RqFTDRH7n26Hl8ANyJHPO34vuOP8cto0pMl8zoYep6F9vhdD1uWSKIUjpjWbZXcKqYkDSCqtMtkDPT2VoKzw/w==} + '@marswave/listenhub-sdk@0.0.17': + resolution: {integrity: sha512-DZZw/A4TfsFJZiorptt7+I6FrveSt57eHiy1Ccwqq1FHHxBF1PwO55u0Lls80Fj1GpZ9DdLVJVxCNYlg0td9FA==} engines: {node: '>=20'} '@napi-rs/wasm-runtime@1.1.4': @@ -1118,7 +1118,7 @@ snapshots: '@jridgewell/sourcemap-codec@1.5.5': {} - '@marswave/listenhub-sdk@0.0.16': + '@marswave/listenhub-sdk@0.0.17': dependencies: ky: 1.14.3 diff --git a/source/_shared/image-dimensions.ts b/source/_shared/image-dimensions.ts new file mode 100644 index 0000000..5408c8d --- /dev/null +++ b/source/_shared/image-dimensions.ts @@ -0,0 +1,146 @@ +import {readFile, stat} from 'node:fs/promises'; +import type {VideoReferenceImageMeta, VideoReferenceImageRole} from './video-reference-metadata.js'; + +type ImageDimensions = { + width: number; + height: number; +}; + +function byte(data: Uint8Array, index: number): number { + const value = data[index]; + if (typeof value !== 'number') throw new Error('Cannot read image dimensions: file is truncated'); + return value; +} + +function readUint16BE(data: Uint8Array, offset: number): number { + return (byte(data, offset) << 8) + byte(data, offset + 1); +} + +function readUint16LE(data: Uint8Array, offset: number): number { + return byte(data, offset) + (byte(data, offset + 1) << 8); +} + +function readUint24LE(data: Uint8Array, offset: number): number { + return byte(data, offset) + (byte(data, offset + 1) << 8) + (byte(data, offset + 2) << 16); +} + +function readUint32BE(data: Uint8Array, offset: number): number { + return ( + (byte(data, offset) << 24) + + (byte(data, offset + 1) << 16) + + (byte(data, offset + 2) << 8) + + byte(data, offset + 3) + ); +} + +function matches(data: Uint8Array, offset: number, text: string): boolean { + return Array.from(text).every((char, index) => byte(data, offset + index) === char.charCodeAt(0)); +} + +function parsePng(data: Uint8Array): ImageDimensions | undefined { + if ( + data.length < 24 || + byte(data, 0) !== 0x89 || + !matches(data, 1, 'PNG') || + !matches(data, 12, 'IHDR') + ) { + return undefined; + } + + return {width: readUint32BE(data, 16), height: readUint32BE(data, 20)}; +} + +function parseGif(data: Uint8Array): ImageDimensions | undefined { + if (data.length < 10 || (!matches(data, 0, 'GIF87a') && !matches(data, 0, 'GIF89a'))) { + return undefined; + } + + return {width: readUint16LE(data, 6), height: readUint16LE(data, 8)}; +} + +function parseJpeg(data: Uint8Array): ImageDimensions | undefined { + if (data.length < 4 || byte(data, 0) !== 0xff || byte(data, 1) !== 0xd8) return undefined; + + let offset = 2; + while (offset + 9 < data.length) { + if (byte(data, offset) !== 0xff) { + offset += 1; + continue; + } + + let marker = byte(data, offset + 1); + while (marker === 0xff) { + offset += 1; + marker = byte(data, offset + 1); + } + + if (marker === 0xd9 || marker === 0xda) break; + const length = readUint16BE(data, offset + 2); + if (length < 2) break; + + const isStartOfFrame = + (marker >= 0xc0 && marker <= 0xc3) || + (marker >= 0xc5 && marker <= 0xc7) || + (marker >= 0xc9 && marker <= 0xcb) || + (marker >= 0xcd && marker <= 0xcf); + if (isStartOfFrame) { + return {height: readUint16BE(data, offset + 5), width: readUint16BE(data, offset + 7)}; + } + + offset += 2 + length; + } + + return undefined; +} + +function parseWebp(data: Uint8Array): ImageDimensions | undefined { + if (data.length < 30 || !matches(data, 0, 'RIFF') || !matches(data, 8, 'WEBP')) { + return undefined; + } + + const chunk = String.fromCharCode(byte(data, 12), byte(data, 13), byte(data, 14), byte(data, 15)); + if (chunk === 'VP8X') { + return {width: readUint24LE(data, 24) + 1, height: readUint24LE(data, 27) + 1}; + } + + if (chunk === 'VP8 ' && matches(data, 23, '\u009d\u0001*')) { + return { + width: readUint16LE(data, 26) & 0x3fff, + height: readUint16LE(data, 28) & 0x3fff, + }; + } + + if (chunk === 'VP8L' && byte(data, 20) === 0x2f) { + const b1 = byte(data, 21); + const b2 = byte(data, 22); + const b3 = byte(data, 23); + const b4 = byte(data, 24); + return { + width: 1 + (((b2 & 0x3f) << 8) | b1), + height: 1 + (((b4 & 0x0f) << 10) | (b3 << 2) | ((b2 & 0xc0) >> 6)), + }; + } + + return undefined; +} + +async function readImageDimensions(filePath: string): Promise { + const data = await readFile(filePath); + const dimensions = parsePng(data) ?? parseGif(data) ?? parseJpeg(data) ?? parseWebp(data); + if (!dimensions) { + throw new Error('Cannot read image dimensions: unsupported or invalid image file'); + } + + return dimensions; +} + +export async function readLocalImageMeta( + filePath: string, + role: VideoReferenceImageRole, +): Promise { + const [{width, height}, fileStat] = await Promise.all([ + readImageDimensions(filePath), + stat(filePath), + ]); + return {role, width, height, size: fileStat.size}; +} diff --git a/source/_shared/upload.ts b/source/_shared/upload.ts index 61ae777..e48c1c3 100644 --- a/source/_shared/upload.ts +++ b/source/_shared/upload.ts @@ -1,9 +1,16 @@ import {access, readFile, stat} from 'node:fs/promises'; import path from 'node:path'; -import type {ListenHubClient} from '@marswave/listenhub-sdk'; type FileAcceptType = 'audio' | 'image' | 'video'; +type FileUploadClient = { + createFileUpload(params: { + fileKey: string; + contentType: string; + category: string; + }): Promise<{presignedUrl: string; fileUrl: string}>; +}; + const audioExtensions = new Set(['.mp3', '.wav', '.flac', '.m4a', '.ogg', '.aac']); const imageExtensions = new Set(['.jpg', '.jpeg', '.png', '.webp', '.gif']); const videoExtensions = new Set(['.mp4', '.mov']); @@ -43,7 +50,7 @@ function allowedExtensions(accept: FileAcceptType): Set { } export async function resolveFileOrUrl( - client: ListenHubClient, + client: FileUploadClient, input: string, options: {accept: FileAcceptType; category?: string}, ): Promise { diff --git a/source/_shared/video-reference-metadata.ts b/source/_shared/video-reference-metadata.ts new file mode 100644 index 0000000..d5dcb34 --- /dev/null +++ b/source/_shared/video-reference-metadata.ts @@ -0,0 +1,60 @@ +export type VideoReferenceImageRole = 'first_frame' | 'last_frame' | 'reference_image'; + +export type VideoReferenceImageMeta = { + role: VideoReferenceImageRole; + width: number; + height: number; + size?: number; +}; + +export type VideoReferenceVideoMeta = { + role: 'reference_video'; + width: number; + height: number; + duration?: number; + fps?: number; + size?: number; +}; + +export function parseImageMeta( + value: string, + role: VideoReferenceImageRole, +): VideoReferenceImageMeta { + const match = /^(\d+)x(\d+)(?::(\d+))?$/.exec(value.trim()); + if (!match) { + throw new Error('Image metadata must be WIDTHxHEIGHT[:SIZE], for example 1080x1920:3600000'); + } + + const [, width, height, size] = match; + return { + role, + width: Number(width), + height: Number(height), + ...(size !== undefined && {size: Number(size)}), + }; +} + +export function parseVideoMeta(value: string): VideoReferenceVideoMeta { + const match = /^(\d+)x(\d+)(?::(\d+(?:\.\d+)?))?(?::(\d+(?:\.\d+)?))?(?::(\d+))?$/.exec( + value.trim(), + ); + if (!match) { + throw new Error( + 'Video metadata must be WIDTHxHEIGHT[:DURATION[:FPS[:SIZE]]], for example 1280x720:5:30:8000000', + ); + } + + const [, width, height, duration, fps, size] = match; + return { + role: 'reference_video', + width: Number(width), + height: Number(height), + ...(duration !== undefined && {duration: Number(duration)}), + ...(fps !== undefined && {fps: Number(fps)}), + ...(size !== undefined && {size: Number(size)}), + }; +} + +export function isSeedanceVideoModel(model: string | undefined, defaultModel: string): boolean { + return (model ?? defaultModel).startsWith('doubao-seedance'); +} diff --git a/source/openapi/client.ts b/source/openapi/client.ts index a6737ad..701f498 100644 --- a/source/openapi/client.ts +++ b/source/openapi/client.ts @@ -2,18 +2,25 @@ import {OpenAPIClient} from '@marswave/listenhub-sdk'; import process from 'node:process'; import {loadOpenAPIConfig} from './config.js'; -export async function getOpenAPIClient(): Promise { +const DEFAULT_OPENAPI_BASE_URL = 'https://api.marswave.ai/openapi'; + +export async function getOpenAPIOptions(): Promise<{apiKey: string; baseURL: string}> { const envKey = process.env['LISTENHUB_API_KEY']; + const baseURL = process.env['LISTENHUB_OPENAPI_URL'] || DEFAULT_OPENAPI_BASE_URL; if (envKey) { - return new OpenAPIClient({apiKey: envKey}); + return {apiKey: envKey, baseURL}; } const config = await loadOpenAPIConfig(); if (config?.apiKey) { - return new OpenAPIClient({apiKey: config.apiKey}); + return {apiKey: config.apiKey, baseURL}; } throw new Error( 'No API Key configured. Set LISTENHUB_API_KEY env var or run `listenhub openapi config set-key`.', ); } + +export async function getOpenAPIClient(): Promise { + return new OpenAPIClient(await getOpenAPIOptions()); +} diff --git a/source/openapi/video.ts b/source/openapi/video.ts index d2fc695..abf78bf 100644 --- a/source/openapi/video.ts +++ b/source/openapi/video.ts @@ -1,15 +1,26 @@ +import path from 'node:path'; import type {Command} from 'commander'; import type { OpenAPICreatePixVerseVideoParams, OpenAPICreateVideoGenerationParams, OpenAPIEstimatePixVerseCreditsParams, + OpenAPIEstimateVideoCreditsParams, OpenAPIPixVerseAsset, OpenAPIPixVerseOptions, OpenAPIVideoGenerationTaskDetail, OpenAPIVideoGenerationTaskStatus, } from '@marswave/listenhub-sdk'; import {handleError, printDetail, printJson, printTable} from '../_shared/output.js'; +import {readLocalImageMeta} from '../_shared/image-dimensions.js'; +import {resolveFileOrUrl} from '../_shared/upload.js'; import {normalizeVideoTaskId} from '../_shared/video-task-id.js'; +import { + isSeedanceVideoModel, + parseImageMeta, + parseVideoMeta, + type VideoReferenceImageMeta, + type VideoReferenceVideoMeta, +} from '../_shared/video-reference-metadata.js'; import {getOpenAPIClient} from './client.js'; import {pollOpenAPI} from './polling.js'; @@ -105,9 +116,13 @@ type PixVerseEstimateOptions = { type VideoCreateOptions = { prompt: string; firstFrame?: string; + firstFrameMeta?: string; lastFrame?: string; + lastFrameMeta?: string; referenceImage: string[]; + referenceImageMeta: string[]; referenceVideo: string[]; + referenceVideoMeta: string[]; referenceAudio: string[]; inputVideoDuration?: string; model?: string; @@ -121,6 +136,16 @@ type VideoCreateOptions = { json: boolean; }; +type OpenAPIVideoGenerationParamsWithMetadata = OpenAPICreateVideoGenerationParams & { + referenceImages?: VideoReferenceImageMeta[]; + referenceVideos?: VideoReferenceVideoMeta[]; +}; + +type OpenAPIEstimateVideoCreditsParamsWithMetadata = OpenAPIEstimateVideoCreditsParams & { + referenceImages?: VideoReferenceImageMeta[]; + referenceVideos?: VideoReferenceVideoMeta[]; +}; + type VideoGetOptions = { json: boolean; }; @@ -139,9 +164,113 @@ type VideoEstimateOptions = { ratio?: string; hasVideoInput: boolean; inputVideoDuration?: string; + referenceImageMeta: string[]; + referenceVideoMeta: string[]; json: boolean; }; +function isHttpUrl(value: string): boolean { + return value.startsWith('http://') || value.startsWith('https://'); +} + +async function getReferenceImages( + options: VideoCreateOptions, + defaultModel: string, +): Promise { + const images: VideoReferenceImageMeta[] = []; + const inferLocalImages = isSeedanceVideoModel(options.model, defaultModel); + + if (options.firstFrameMeta !== undefined) { + images.push(parseImageMeta(options.firstFrameMeta, 'first_frame')); + } else if ( + inferLocalImages && + options.firstFrame !== undefined && + !isHttpUrl(options.firstFrame) + ) { + images.push(await readLocalImageMeta(path.resolve(options.firstFrame.trim()), 'first_frame')); + } + + if (options.lastFrameMeta !== undefined) { + images.push(parseImageMeta(options.lastFrameMeta, 'last_frame')); + } else if (inferLocalImages && options.lastFrame !== undefined && !isHttpUrl(options.lastFrame)) { + images.push(await readLocalImageMeta(path.resolve(options.lastFrame.trim()), 'last_frame')); + } + + const referenceImages = await Promise.all( + options.referenceImage.map(async (ref, index) => { + const meta = options.referenceImageMeta[index]; + if (meta !== undefined) return parseImageMeta(meta, 'reference_image'); + if (inferLocalImages && !isHttpUrl(ref)) { + return readLocalImageMeta(path.resolve(ref.trim()), 'reference_image'); + } + + return undefined; + }), + ); + images.push( + ...referenceImages.filter((image): image is VideoReferenceImageMeta => image !== undefined), + ); + + return images; +} + +function getReferenceVideos(options: VideoCreateOptions): VideoReferenceVideoMeta[] { + return options.referenceVideoMeta.map(parseVideoMeta); +} + +function validateReferenceMetadata(options: VideoCreateOptions): void { + if (options.firstFrameMeta !== undefined && options.firstFrame === undefined) { + throw new Error('--first-frame-meta requires --first-frame'); + } + if (options.lastFrameMeta !== undefined && options.lastFrame === undefined) { + throw new Error('--last-frame-meta requires --last-frame'); + } + if ( + options.referenceImageMeta.length > 0 && + options.referenceImageMeta.length > options.referenceImage.length + ) { + throw new Error('--reference-image-meta count cannot exceed --reference-image count'); + } + if ( + options.referenceVideoMeta.length > 0 && + options.referenceVideoMeta.length !== options.referenceVideo.length + ) { + throw new Error('--reference-video-meta count must match --reference-video count'); + } + + if (!isSeedanceVideoModel(options.model, 'doubao-seedance-2-fast')) { + return; + } + + if ( + options.firstFrame !== undefined && + options.firstFrameMeta === undefined && + isHttpUrl(options.firstFrame) + ) { + throw new Error('Seedance --first-frame requires --first-frame-meta WIDTHxHEIGHT[:SIZE]'); + } + if ( + options.lastFrame !== undefined && + options.lastFrameMeta === undefined && + isHttpUrl(options.lastFrame) + ) { + throw new Error('Seedance --last-frame requires --last-frame-meta WIDTHxHEIGHT[:SIZE]'); + } + if ( + options.referenceImage.some( + (ref, index) => options.referenceImageMeta[index] === undefined && isHttpUrl(ref), + ) + ) { + throw new Error('Seedance URL --reference-image requires one --reference-image-meta per image'); + } + if ( + options.referenceVideo.length > 0 && + options.referenceVideoMeta.length !== options.referenceVideo.length + ) { + throw new Error('Seedance --reference-video requires one --reference-video-meta per video'); + } +} + function printVideoDetail(result: OpenAPIVideoGenerationTaskDetail): void { printDetail('Video Generation Task', [ ['ID', result.id], @@ -164,23 +293,37 @@ export function register(openapi: Command) { .command('create') .description('Create a video generation task') .requiredOption('--prompt ', 'Video description / prompt') - .option('--first-frame ', 'First frame image URL') - .option('--last-frame ', 'Last frame image URL (requires --first-frame)') + .option('--first-frame ', 'First frame image') + .option('--first-frame-meta ', 'First frame metadata WIDTHxHEIGHT[:SIZE]') + .option('--last-frame ', 'Last frame image (requires --first-frame)') + .option('--last-frame-meta ', 'Last frame metadata WIDTHxHEIGHT[:SIZE]') + .option( + '--reference-image ', + 'Reference image (repeatable, max 9)', + collect, + [] as string[], + ) .option( - '--reference-image ', - 'Reference image URL (repeatable, max 9)', + '--reference-image-meta ', + 'Reference image metadata WIDTHxHEIGHT[:SIZE] (repeatable, same order)', collect, [] as string[], ) .option( - '--reference-video ', - 'Reference video URL (repeatable, max 3)', + '--reference-video ', + 'Reference video (repeatable, max 3)', collect, [] as string[], ) .option( - '--reference-audio ', - 'Reference audio URL (repeatable, max 3)', + '--reference-video-meta ', + 'Reference video metadata WIDTHxHEIGHT[:DURATION[:FPS[:SIZE]]] (repeatable, same order)', + collect, + [] as string[], + ) + .option( + '--reference-audio ', + 'Reference audio (repeatable, max 3)', collect, [] as string[], ) @@ -203,6 +346,7 @@ export function register(openapi: Command) { if (options.lastFrame && !options.firstFrame) { throw new Error('--last-frame requires --first-frame'); } + validateReferenceMetadata(options); const hasFrameMode = Boolean(options.firstFrame) || Boolean(options.lastFrame); const hasReferenceMode = @@ -265,40 +409,64 @@ export function register(openapi: Command) { } } + const client = await getOpenAPIClient(); + // Build content array const content: OpenAPICreateVideoGenerationParams['content'] = [ {type: 'text', text: options.prompt}, ]; if (options.firstFrame) { + const url = await resolveFileOrUrl(client, options.firstFrame, { + accept: 'image', + category: 'episode', + }); content.push({ type: 'image_url', - image_url: {url: options.firstFrame}, + image_url: {url}, role: 'first_frame', }); } if (options.lastFrame) { + const url = await resolveFileOrUrl(client, options.lastFrame, { + accept: 'image', + category: 'episode', + }); content.push({ type: 'image_url', - image_url: {url: options.lastFrame}, + image_url: {url}, role: 'last_frame', }); } - for (const url of options.referenceImage) { + for (const ref of options.referenceImage) { + const url = await resolveFileOrUrl(client, ref, { + accept: 'image', + category: 'episode', + }); // eslint-disable-line no-await-in-loop content.push({type: 'image_url', image_url: {url}, role: 'reference_image'}); } - for (const url of options.referenceVideo) { + for (const ref of options.referenceVideo) { + const url = await resolveFileOrUrl(client, ref, { + accept: 'video', + category: 'episode', + }); // eslint-disable-line no-await-in-loop content.push({type: 'video_url', video_url: {url}, role: 'reference_video'}); } - for (const url of options.referenceAudio) { + for (const ref of options.referenceAudio) { + const url = await resolveFileOrUrl(client, ref, { + accept: 'audio', + category: 'episode', + }); // eslint-disable-line no-await-in-loop content.push({type: 'audio_url', audio_url: {url}, role: 'reference_audio'}); } - const params: OpenAPICreateVideoGenerationParams = { + const referenceImages = await getReferenceImages(options, 'doubao-seedance-2-fast'); + const referenceVideos = getReferenceVideos(options); + const params: OpenAPIVideoGenerationParamsWithMetadata = { content, model: options.model as OpenAPICreateVideoGenerationParams['model'], resolution: options.resolution as OpenAPICreateVideoGenerationParams['resolution'], @@ -310,9 +478,10 @@ export function register(openapi: Command) { options.inputVideoDuration === undefined ? undefined : Number(options.inputVideoDuration), + ...(referenceImages.length > 0 && {referenceImages}), + ...(referenceVideos.length > 0 && {referenceVideos}), }; - const client = await getOpenAPIClient(); const {taskId: rawTaskId} = await client.createVideoGeneration(params); const taskId = normalizeVideoTaskId(rawTaskId); @@ -420,6 +589,18 @@ export function register(openapi: Command) { .option('--ratio ', 'Aspect ratio: 16:9, 4:3, 1:1, 3:4, 9:16, 21:9') .option('--has-video-input', 'Whether input video is provided', false) .option('--input-video-duration ', 'Input video duration in seconds', Number) + .option( + '--reference-image-meta ', + 'Reference image metadata WIDTHxHEIGHT[:SIZE] for estimate', + collect, + [] as string[], + ) + .option( + '--reference-video-meta ', + 'Reference video metadata WIDTHxHEIGHT[:DURATION[:FPS[:SIZE]]] for estimate', + collect, + [] as string[], + ) .option('-j, --json', 'Output JSON', false) .action(async (options: VideoEstimateOptions) => { try { @@ -428,7 +609,11 @@ export function register(openapi: Command) { } const client = await getOpenAPIClient(); - const result = await client.estimateVideoCredits({ + const referenceImages = options.referenceImageMeta.map((meta) => + parseImageMeta(meta, 'reference_image'), + ); + const referenceVideos = options.referenceVideoMeta.map(parseVideoMeta); + const params: OpenAPIEstimateVideoCreditsParamsWithMetadata = { model: options.model as OpenAPICreateVideoGenerationParams['model'] extends infer M ? NonNullable : never, @@ -440,7 +625,10 @@ export function register(openapi: Command) { options.inputVideoDuration === undefined ? undefined : Number(options.inputVideoDuration), - }); + ...(referenceImages.length > 0 && {referenceImages}), + ...(referenceVideos.length > 0 && {referenceVideos}), + }; + const result = await client.estimateVideoCredits(params); if (options.json) { printJson(result); diff --git a/source/video/_cli.ts b/source/video/_cli.ts index 790d811..4f2cde5 100644 --- a/source/video/_cli.ts +++ b/source/video/_cli.ts @@ -31,9 +31,23 @@ export function register(program: Command) { .option('--ratio ', 'Aspect ratio: 16:9, 4:3, 1:1, 3:4, 9:16, 21:9, 4:5, 5:4') .option('--duration ', 'Video duration in seconds (3-15)', Number) .option('--first-frame ', 'First frame image') + .option('--first-frame-meta ', 'First frame metadata WIDTHxHEIGHT[:SIZE]') .option('--last-frame ', 'Last frame image (requires --first-frame)') + .option('--last-frame-meta ', 'Last frame metadata WIDTHxHEIGHT[:SIZE]') .option('--reference-image ', 'Reference image (repeatable, max 9)', collect, []) + .option( + '--reference-image-meta ', + 'Reference image metadata WIDTHxHEIGHT[:SIZE] (repeatable, same order)', + collect, + [], + ) .option('--reference-video ', 'Reference video (repeatable, max 3)', collect, []) + .option( + '--reference-video-meta ', + 'Reference video metadata WIDTHxHEIGHT[:DURATION[:FPS[:SIZE]]] (repeatable, same order)', + collect, + [], + ) .option('--reference-audio ', 'Reference audio (repeatable, max 3)', collect, []) .option( '--input-video-duration ', @@ -93,6 +107,18 @@ export function register(program: Command) { .option('--ratio ', 'Aspect ratio', '16:9') .option('--has-video-input', 'Has reference video input', false) .option('--input-video-duration ', 'Reference video duration', Number) + .option( + '--reference-image-meta ', + 'Reference image metadata WIDTHxHEIGHT[:SIZE] for estimate', + collect, + [], + ) + .option( + '--reference-video-meta ', + 'Reference video metadata WIDTHxHEIGHT[:DURATION[:FPS[:SIZE]]] for estimate', + collect, + [], + ) .option('-j, --json', 'Output JSON', false) .action(async (options: VideoEstimateOptions) => { try { diff --git a/source/video/video.ts b/source/video/video.ts index bb37d57..98f6a0d 100644 --- a/source/video/video.ts +++ b/source/video/video.ts @@ -14,6 +14,14 @@ import {printDetail, printJson, printTable} from '../_shared/output.js'; import {pollVideoTaskUntilDone} from '../_shared/polling.js'; import {resolveFileOrUrl} from '../_shared/upload.js'; import {normalizeVideoTaskId} from '../_shared/video-task-id.js'; +import {readLocalImageMeta} from '../_shared/image-dimensions.js'; +import { + isSeedanceVideoModel, + parseImageMeta, + parseVideoMeta, + type VideoReferenceImageMeta, + type VideoReferenceVideoMeta, +} from '../_shared/video-reference-metadata.js'; export type VideoCreateOptions = { prompt: string; @@ -22,9 +30,13 @@ export type VideoCreateOptions = { ratio?: string; duration?: number; firstFrame?: string; + firstFrameMeta?: string; lastFrame?: string; + lastFrameMeta?: string; referenceImage: string[]; + referenceImageMeta: string[]; referenceVideo: string[]; + referenceVideoMeta: string[]; referenceAudio: string[]; inputVideoDuration?: number; generateAudio: boolean; @@ -35,6 +47,11 @@ export type VideoCreateOptions = { json: boolean; }; +type CreateVideoGenerationParamsWithMetadata = CreateVideoGenerationParams & { + referenceImages?: VideoReferenceImageMeta[]; + referenceVideos?: VideoReferenceVideoMeta[]; +}; + export type VideoListOptions = { page: number; pageSize: number; @@ -49,12 +66,121 @@ export type VideoEstimateOptions = { ratio: string; hasVideoInput: boolean; inputVideoDuration?: number; + referenceImageMeta: string[]; + referenceVideoMeta: string[]; json: boolean; }; +type EstimateVideoGenerationCreditsParamsWithMetadata = EstimateVideoGenerationCreditsParams & { + referenceImages?: VideoReferenceImageMeta[]; + referenceVideos?: VideoReferenceVideoMeta[]; +}; + const allowedVideoAudioExtensions = new Set(['.mp3', '.wav']); const allowedVideoExtensions = new Set(['.mp4', '.mov']); +function isHttpUrl(value: string): boolean { + return value.startsWith('http://') || value.startsWith('https://'); +} + +async function getReferenceImages( + options: VideoCreateOptions, + defaultModel: string, +): Promise { + const images: VideoReferenceImageMeta[] = []; + const inferLocalImages = isSeedanceVideoModel(options.model, defaultModel); + + if (options.firstFrameMeta !== undefined) { + images.push(parseImageMeta(options.firstFrameMeta, 'first_frame')); + } else if ( + inferLocalImages && + options.firstFrame !== undefined && + !isHttpUrl(options.firstFrame) + ) { + images.push(await readLocalImageMeta(path.resolve(options.firstFrame.trim()), 'first_frame')); + } + + if (options.lastFrameMeta !== undefined) { + images.push(parseImageMeta(options.lastFrameMeta, 'last_frame')); + } else if (inferLocalImages && options.lastFrame !== undefined && !isHttpUrl(options.lastFrame)) { + images.push(await readLocalImageMeta(path.resolve(options.lastFrame.trim()), 'last_frame')); + } + + const referenceImages = await Promise.all( + options.referenceImage.map(async (ref, index) => { + const meta = options.referenceImageMeta[index]; + if (meta !== undefined) return parseImageMeta(meta, 'reference_image'); + if (inferLocalImages && !isHttpUrl(ref)) { + return readLocalImageMeta(path.resolve(ref.trim()), 'reference_image'); + } + + return undefined; + }), + ); + images.push( + ...referenceImages.filter((image): image is VideoReferenceImageMeta => image !== undefined), + ); + + return images; +} + +function getReferenceVideos(options: VideoCreateOptions): VideoReferenceVideoMeta[] { + return options.referenceVideoMeta.map(parseVideoMeta); +} + +function validateReferenceMetadata(options: VideoCreateOptions): void { + if (options.firstFrameMeta !== undefined && options.firstFrame === undefined) { + throw new Error('--first-frame-meta requires --first-frame'); + } + if (options.lastFrameMeta !== undefined && options.lastFrame === undefined) { + throw new Error('--last-frame-meta requires --last-frame'); + } + if ( + options.referenceImageMeta.length > 0 && + options.referenceImageMeta.length > options.referenceImage.length + ) { + throw new Error('--reference-image-meta count cannot exceed --reference-image count'); + } + if ( + options.referenceVideoMeta.length > 0 && + options.referenceVideoMeta.length !== options.referenceVideo.length + ) { + throw new Error('--reference-video-meta count must match --reference-video count'); + } + + if (!isSeedanceVideoModel(options.model, 'happyhorse')) { + return; + } + + if ( + options.firstFrame !== undefined && + options.firstFrameMeta === undefined && + isHttpUrl(options.firstFrame) + ) { + throw new Error('Seedance --first-frame requires --first-frame-meta WIDTHxHEIGHT[:SIZE]'); + } + if ( + options.lastFrame !== undefined && + options.lastFrameMeta === undefined && + isHttpUrl(options.lastFrame) + ) { + throw new Error('Seedance --last-frame requires --last-frame-meta WIDTHxHEIGHT[:SIZE]'); + } + if ( + options.referenceImage.some( + (ref, index) => options.referenceImageMeta[index] === undefined && isHttpUrl(ref), + ) + ) { + throw new Error('Seedance URL --reference-image requires one --reference-image-meta per image'); + } + if ( + options.referenceVideo.length > 0 && + options.referenceVideoMeta.length !== options.referenceVideo.length + ) { + throw new Error('Seedance --reference-video requires one --reference-video-meta per video'); + } +} + function validateCreateOptions(options: VideoCreateOptions): void { if (options.duration !== undefined && (options.duration < 3 || options.duration > 15)) { throw new Error('Duration must be between 3 and 15 seconds'); @@ -67,6 +193,7 @@ function validateCreateOptions(options: VideoCreateOptions): void { if (options.lastFrame && !options.firstFrame) { throw new Error('--last-frame requires --first-frame'); } + validateReferenceMetadata(options); const hasFrameMode = Boolean(options.firstFrame || options.lastFrame); const hasReferenceMode = @@ -195,7 +322,9 @@ export async function createVideo( content.push({type: 'audio_url', audio_url: {url}, role: 'reference_audio'}); } - const params: CreateVideoGenerationParams = { + const referenceImages = await getReferenceImages(options, 'happyhorse'); + const referenceVideos = getReferenceVideos(options); + const params: CreateVideoGenerationParamsWithMetadata = { content, ...(options.model && {model: options.model as VideoGenerationModel}), ...(options.resolution && {resolution: options.resolution as VideoGenerationResolution}), @@ -207,6 +336,8 @@ export async function createVideo( inputVideoDuration: options.inputVideoDuration, }), ...(options.audioSetting && {audioSetting: options.audioSetting as 'auto' | 'origin'}), + ...(referenceImages.length > 0 && {referenceImages}), + ...(referenceVideos.length > 0 && {referenceVideos}), }; const {taskId: rawTaskId} = await client.createVideoGeneration(params); @@ -317,7 +448,16 @@ export async function estimateCredits( }), }; - const result = await client.estimateVideoGenerationCredits(params); + const referenceImages = options.referenceImageMeta.map((meta) => + parseImageMeta(meta, 'reference_image'), + ); + const referenceVideos = options.referenceVideoMeta.map(parseVideoMeta); + const estimateParams: EstimateVideoGenerationCreditsParamsWithMetadata = { + ...params, + ...(referenceImages.length > 0 && {referenceImages}), + ...(referenceVideos.length > 0 && {referenceVideos}), + }; + const result = await client.estimateVideoGenerationCredits(estimateParams); if (options.json) { printJson(result); diff --git a/tests/openapi/client.test.ts b/tests/openapi/client.test.ts index 75f67bd..ffa3493 100644 --- a/tests/openapi/client.test.ts +++ b/tests/openapi/client.test.ts @@ -15,8 +15,12 @@ describe('getOpenAPIClient', () => { tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'lh-cli-test-')); vi.stubEnv('XDG_CONFIG_HOME', tmpDir); vi.stubEnv('LISTENHUB_API_KEY', ''); + vi.stubEnv('LISTENHUB_OPENAPI_URL', ''); mockOpenAPIClient.mockClear(); - mockOpenAPIClient.mockImplementation(function (this: {apiKey: string}, opts: {apiKey: string}) { + mockOpenAPIClient.mockImplementation(function ( + this: {apiKey: string}, + opts: {apiKey: string; baseURL: string}, + ) { this.apiKey = opts.apiKey; }); }); @@ -35,7 +39,21 @@ describe('getOpenAPIClient', () => { vi.stubEnv('LISTENHUB_API_KEY', 'lh_sk_env_secret'); const {getOpenAPIClient} = await import('../../source/openapi/client.js'); await getOpenAPIClient(); - expect(mockOpenAPIClient).toHaveBeenCalledWith({apiKey: 'lh_sk_env_secret'}); + expect(mockOpenAPIClient).toHaveBeenCalledWith({ + apiKey: 'lh_sk_env_secret', + baseURL: 'https://api.marswave.ai/openapi', + }); + }); + + it('passes LISTENHUB_OPENAPI_URL to OpenAPIClient', async () => { + vi.stubEnv('LISTENHUB_API_KEY', 'lh_sk_env_secret'); + vi.stubEnv('LISTENHUB_OPENAPI_URL', 'https://staging.example.com/openapi'); + const {getOpenAPIClient} = await import('../../source/openapi/client.js'); + await getOpenAPIClient(); + expect(mockOpenAPIClient).toHaveBeenCalledWith({ + apiKey: 'lh_sk_env_secret', + baseURL: 'https://staging.example.com/openapi', + }); }); it('uses config file when env var is absent and passes file key', async () => { @@ -48,7 +66,10 @@ describe('getOpenAPIClient', () => { ); const {getOpenAPIClient} = await import('../../source/openapi/client.js'); await getOpenAPIClient(); - expect(mockOpenAPIClient).toHaveBeenCalledWith({apiKey: 'lh_sk_file_secret'}); + expect(mockOpenAPIClient).toHaveBeenCalledWith({ + apiKey: 'lh_sk_file_secret', + baseURL: 'https://api.marswave.ai/openapi', + }); }); it('env var takes priority over config file', async () => { @@ -60,6 +81,9 @@ describe('getOpenAPIClient', () => { }); const {getOpenAPIClient} = await import('../../source/openapi/client.js'); await getOpenAPIClient(); - expect(mockOpenAPIClient).toHaveBeenCalledWith({apiKey: 'lh_sk_env_priority'}); + expect(mockOpenAPIClient).toHaveBeenCalledWith({ + apiKey: 'lh_sk_env_priority', + baseURL: 'https://api.marswave.ai/openapi', + }); }); }); diff --git a/tests/openapi/commands.test.ts b/tests/openapi/commands.test.ts index 6994b25..dce5b81 100644 --- a/tests/openapi/commands.test.ts +++ b/tests/openapi/commands.test.ts @@ -1,3 +1,6 @@ +import {mkdtemp, rm, writeFile} from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; import {afterEach, beforeEach, describe, expect, it, vi} from 'vitest'; import {Command} from 'commander'; import {register as registerSpeakers} from '../../source/openapi/speakers.js'; @@ -24,6 +27,7 @@ const mockClient = vi.hoisted(() => ({ getVideoGenerationTask: vi.fn(), listVideoGenerationTasks: vi.fn(), estimateVideoCredits: vi.fn(), + createFileUpload: vi.fn(), createPixVerseVideoGeneration: vi.fn(), estimatePixVerseVideoCredits: vi.fn(), createContentExtract: vi.fn(), @@ -54,8 +58,38 @@ beforeEach(() => { afterEach(() => { vi.restoreAllMocks(); + vi.unstubAllGlobals(); }); +function pngBytes(width: number, height: number) { + return new Uint8Array([ + 0x89, + 0x50, + 0x4e, + 0x47, + 0x0d, + 0x0a, + 0x1a, + 0x0a, + 0x00, + 0x00, + 0x00, + 0x0d, + 0x49, + 0x48, + 0x44, + 0x52, + (width >>> 24) & 0xff, + (width >>> 16) & 0xff, + (width >>> 8) & 0xff, + width & 0xff, + (height >>> 24) & 0xff, + (height >>> 16) & 0xff, + (height >>> 8) & 0xff, + height & 0xff, + ]); +} + describe('speakers list', () => { it('calls listSpeakers without filter and prints JSON', async () => { const items = [{name: 'Alice', speakerId: 'alice-01', gender: 'female', language: 'en'}]; @@ -189,6 +223,8 @@ describe('video create', () => { 'Timelapse', '--first-frame', 'https://img.example.com/frame.jpg', + '--first-frame-meta', + '1080x1920:3600000', '--no-wait', '--json', ], @@ -205,8 +241,173 @@ describe('video create', () => { role: 'first_frame', }, ], + referenceImages: [{role: 'first_frame', width: 1080, height: 1920, size: 3_600_000}], + }), + ); + }); + + it('uploads a local first-frame image and auto-populates reference metadata', async () => { + const tmpDir = await mkdtemp(path.join(os.tmpdir(), 'listenhub-cli-video-')); + const framePath = path.join(tmpDir, 'frame.png'); + await writeFile(framePath, pngBytes(1080, 1920)); + const fetchMock = vi + .fn() + .mockResolvedValue(new Response(null, {status: 200, statusText: 'OK'})); + vi.stubGlobal('fetch', fetchMock); + mockClient.createFileUpload.mockResolvedValue({ + presignedUrl: 'https://upload.example.com/frame.png', + fileUrl: 'https://storage.googleapis.com/private-bucket/uploads/frame.png', + }); + mockClient.createVideoGeneration.mockResolvedValue({taskId: '6a201660b9fc373811288f11'}); + vi.spyOn(console, 'log').mockImplementation(() => undefined); + + try { + const parent = makeParent(); + registerVideo(parent); + + await parent.parseAsync( + [ + 'video', + 'create', + '--prompt', + 'Timelapse', + '--first-frame', + framePath, + '--no-wait', + '--json', + ], + {from: 'user'}, + ); + + expect(mockClient.createFileUpload).toHaveBeenCalledWith({ + fileKey: 'frame.png', + contentType: 'image/png', + category: 'episode', + }); + expect(fetchMock).toHaveBeenCalledWith( + 'https://upload.example.com/frame.png', + expect.objectContaining({ + method: 'PUT', + headers: expect.objectContaining({'Content-Type': 'image/png'}), + }), + ); + expect(mockClient.createVideoGeneration).toHaveBeenCalledWith( + expect.objectContaining({ + content: [ + {type: 'text', text: 'Timelapse'}, + { + type: 'image_url', + image_url: {url: 'https://storage.googleapis.com/private-bucket/uploads/frame.png'}, + role: 'first_frame', + }, + ], + referenceImages: [{role: 'first_frame', width: 1080, height: 1920, size: 24}], + }), + ); + } finally { + await rm(tmpDir, {recursive: true, force: true}); + } + }); + + it('includes reference video metadata when provided', async () => { + mockClient.createVideoGeneration.mockResolvedValue({taskId: '6a201660b9fc373811288f10'}); + vi.spyOn(console, 'log').mockImplementation(() => undefined); + + const parent = makeParent(); + registerVideo(parent); + + await parent.parseAsync( + [ + 'video', + 'create', + '--prompt', + 'Restyle this clip', + '--reference-video', + 'https://video.example.com/ref.mp4', + '--reference-video-meta', + '1280x720:5:30:8000000', + '--input-video-duration', + '5', + '--no-wait', + '--json', + ], + {from: 'user'}, + ); + + expect(mockClient.createVideoGeneration).toHaveBeenCalledWith( + expect.objectContaining({ + content: [ + {type: 'text', text: 'Restyle this clip'}, + { + type: 'video_url', + video_url: {url: 'https://video.example.com/ref.mp4'}, + role: 'reference_video', + }, + ], + inputVideoDuration: 5, + referenceVideos: [ + { + role: 'reference_video', + width: 1280, + height: 720, + duration: 5, + fps: 30, + size: 8_000_000, + }, + ], + }), + ); + }); +}); + +describe('video estimate', () => { + it('passes reference metadata to estimateVideoCredits', async () => { + mockClient.estimateVideoCredits.mockResolvedValue({tokens: 3320, credits: 10}); + const consoleSpy = vi.spyOn(console, 'log').mockImplementation(() => undefined); + + const parent = makeParent(); + registerVideo(parent); + + await parent.parseAsync( + [ + 'video', + 'estimate', + '--model', + 'doubao-seedance-2-pro', + '--resolution', + '720p', + '--duration', + '5', + '--has-video-input', + '--input-video-duration', + '5', + '--reference-video-meta', + '1280x720:5:30:8000000', + '--json', + ], + {from: 'user'}, + ); + + expect(mockClient.estimateVideoCredits).toHaveBeenCalledWith( + expect.objectContaining({ + model: 'doubao-seedance-2-pro', + resolution: '720p', + duration: 5, + hasVideoInput: true, + inputVideoDuration: 5, + referenceVideos: [ + { + role: 'reference_video', + width: 1280, + height: 720, + duration: 5, + fps: 30, + size: 8_000_000, + }, + ], }), ); + expect(consoleSpy).toHaveBeenCalledWith(JSON.stringify({tokens: 3320, credits: 10}, null, 2)); }); });