diff --git a/assets/agent/openchatcut-tool-schemas.json b/assets/agent/openchatcut-tool-schemas.json index 8624a30e..8d0b6adc 100644 --- a/assets/agent/openchatcut-tool-schemas.json +++ b/assets/agent/openchatcut-tool-schemas.json @@ -1627,61 +1627,70 @@ }, { "name": "submit_sound", - "description": "Generate one original/custom sound effect with ElevenLabs and create an audio asset in the media pool. Does not place timeline items. For ordinary whooshes, clicks, impacts, dings, and similar editing sounds, use the existing library first.", + "description": "Generate one sound-effect audio asset in the media pool: ElevenLabs synthesizes from a text prompt; Sonilo generates SFX from a project video asset (the rendered cut, up to 3 minutes) with no prompt. Does not place timeline items. For ordinary whooshes, clicks, impacts, dings, and similar editing sounds, use the existing library first.", "input_schema": { "type": "object", "properties": { + "provider": { + "type": "string", + "enum": [ + "elevenlabs", + "sonilo" + ], + "description": "Defaults to elevenlabs (text prompt). sonilo generates video-matched SFX from sourceAssetId instead of a prompt." + }, "prompt": { "type": "string", - "minLength": 1, - "description": "Detailed sound description." + "description": "Detailed sound description. Required for elevenlabs; not used by sonilo." }, "durationSeconds": { "type": "number", "minimum": 0.5, "maximum": 30, - "description": "Optional 0.5–30 seconds. Omit to let ElevenLabs choose the duration." + "description": "ElevenLabs only. Optional 0.5–30 seconds; omit to let ElevenLabs choose the duration." }, "promptInfluence": { "type": "number", "minimum": 0, "maximum": 1, - "description": "Prompt adherence; defaults to 0.3." + "description": "ElevenLabs only. Prompt adherence; defaults to 0.3." }, "loop": { "type": "boolean", - "description": "Create a seamless loop. Requires the configured eleven_text_to_sound_v2 model." + "description": "ElevenLabs only. Create a seamless loop; requires the configured eleven_text_to_sound_v2 model." }, "outputFormat": { "type": "string", - "description": "Official ElevenLabs codec_sample-rate_bitrate enum, e.g. mp3_44100_128, opus_48000_128, pcm_44100." + "description": "ElevenLabs only. Official codec_sample-rate_bitrate enum, e.g. mp3_44100_128, opus_48000_128, pcm_44100." + }, + "sourceAssetId": { + "type": "string", + "description": "Sonilo only. Project video asset (the rendered cut, ≤3 minutes) the SFX are generated from." }, "name": { "type": "string", "description": "Optional media-pool asset name." } - }, - "required": [ - "prompt" - ] + } } }, { "name": "submit_music", - "description": "Submit a Mureka or MiniMax music job. Mureka supports instrumental, lyrics song, prompt song, soundtrack, and track/stem modes with up to 3 results. MiniMax supports text-to-music and cover. Results are saved to the media pool.", + "description": "Submit a Mureka, MiniMax, or Sonilo music job. Mureka supports instrumental, lyrics song, prompt song, soundtrack, and track/stem modes with up to 3 results. MiniMax supports text-to-music and cover. Sonilo v2m generates music from a project video asset (the rendered cut, up to 6 minutes) — video-conditioned, with a single optional style prompt. Results are saved to the media pool.", "input_schema": { "type": "object", "properties": { "prompt": { "type": "string", "maxLength": 2000, - "description": "Style/control prompt. Required for most modes; optional with Mureka provider IDs. Mureka max is 1024 or 2000 by mode; MiniMax cover requires 10–300." + "description": "Style/control prompt. Required for most modes; optional with Mureka provider IDs and Sonilo v2m (≤500, works promptless). Mureka max is 1024 or 2000 by mode; MiniMax cover requires 10–300." }, "provider": { "type": "string", "enum": [ "mureka", - "minimax" + "minimax", + "sonilo" ], "description": "Defaults to mureka." }, @@ -1694,9 +1703,10 @@ "soundtrack", "track", "t2m", - "cover" + "cover", + "v2m" ], - "description": "Mureka: instrumental/song/prompt-song/soundtrack/track. MiniMax: t2m/cover." + "description": "Mureka: instrumental/song/prompt-song/soundtrack/track. MiniMax: t2m/cover. Sonilo: v2m (video-to-music)." }, "lyrics": { "type": "string", @@ -1795,7 +1805,7 @@ }, "sourceAssetId": { "type": "string", - "description": "Mureka soundtrack image/video asset or track-mode audio asset." + "description": "Mureka soundtrack image/video asset, track-mode audio asset, or Sonilo v2m video asset (the rendered cut, ≤6 minutes)." }, "audioStartMs": { "type": "integer", diff --git a/package.json b/package.json index d69ea104..a3928801 100644 --- a/package.json +++ b/package.json @@ -48,7 +48,7 @@ "verify:toolbar-tooltips": "tsx src/media/media-toolbar-tooltips.verify.tsx && tsx src/components/generation-activity.verify.tsx && tsx src/components/topbar-tooltips.verify.tsx", "verify:export-adaptive": "npx tsx src/export/exportDestination.verify.ts && npx tsx src/export/exportRoutePlanner.verify.ts && npx tsx src/export/materializeBlobMedia.verify.ts && npx tsx src/export/serverExportOperation.verify.ts && npx tsx server/plugins/export-destination.verify.ts && npx tsx server/plugins/export-stage.verify.ts && npx tsx server/plugins/export-media-plan.verify.ts && npx tsx server/plugins/export-product-assets.verify.ts && npx tsx server/plugins/export-submission.verify.ts", "test": "node scripts/run-tests.mjs", - "test:serial": "node remotion/performance.verify.mjs && tsx src/transcript/local-asr-warmup.verify.ts && tsx src/transcript/assemblyai.verify.ts && tsx src/transcript/assemblyai-resume.verify.ts && tsx src/transcript/transcribe-jobs.verify.ts && tsx src/persist/mediaIntelligenceStores.verify.ts && tsx src/editor/rateStretch.verify.ts && tsx server/media-acceleration.verify.ts && tsx server/plugins/scene-detection.verify.ts && tsx server/plugins/stock.verify.ts && tsx src/agent/proposal.verify.ts && tsx src/agent/external-edit-session.verify.ts && tsx src/components/chat/widget-parse.verify.ts && tsx src/agent/tools/generation-tool-input.verify.ts && tsx server/plugins/image.verify.ts && tsx server/plugins/video.verify.ts && tsx server/plugins/music.verify.ts && tsx server/plugins/music-media.verify.ts && tsx server/plugins/sound.verify.ts && tsx server/plugins/voice.verify.ts && tsx src/agent/tools/followup-tools.verify.ts && tsx src/agent/tools/clean-script-rules.verify.ts && tsx src/agent/tools/mg-export-params.verify.ts && tsx src/agent/tools/scene-detection-tools.verify.ts && tsx src/agent/tools/isolate-voice-tools.verify.ts && tsx src/agent/tools/template-tools.verify.ts && tsx src/agent/tools/script-tools.verify.ts && tsx src/agent/tools/timeline-target-tools.verify.ts && tsx src/agent/tools/track-tools.verify.ts && tsx server/plugins/generation-jobs.verify.ts && tsx server/plugins/export-runtime.verify.ts && tsx server/plugins/export-params.verify.ts && tsx src/export/browserExport.verify.ts && tsx src/agent/ai-sdk.verify.ts && tsx server/llm-proxy.verify.ts && tsx server/external-agent/broker.verify.ts && tsx server/plugins/project-store.verify.ts && tsx server/plugins/extension-store.verify.ts && tsx server/plugins/normalize-media.verify.ts && tsx src/persist/projectStore.verify.ts && tsx src/persist/design-style-metadata.verify.ts && tsx src/persist/designStyleTransfer.verify.ts && tsx src/captions/manualCaptions.verify.ts && tsx src/captions/captionPreviewTarget.verify.ts && tsx src/agent/tools/captions-track-order.verify.ts && tsx src/skins.verify.ts && tsx src/theme-tokenization.verify.ts && tsx server/mobile-upload-service.verify.ts && tsx src/media/mobileUploadApi.verify.ts && tsx src/tracking/tracking.verify.ts && tsx src/media/semantic-search/semanticSearch.verify.ts && tsx src/color/autoGrade.verify.ts && tsx server/plugins/auto-grade.verify.ts && tsx src/transcript/phrases.verify.ts && tsx src/agent/tools/transcript-read.verify.ts && tsx src/export/quality.verify.ts && tsx src/export/autoQa.verify.ts && tsx server/plugins/export-qa.verify.ts && tsx src/agent/tools/export-qa-tools.verify.ts && tsx src/persist/migrations/migrations.verify.ts && tsx server/keystore.verify.ts && tsx src/components/timeline/transition-context-menu.verify.ts && tsx shared/model-capabilities.verify.ts && tsx desktop/agent-path-import.verify.ts && tsx desktop/agent-path-import.e2e.verify.ts && tsx src/agent/tools/agent-path-import-tools.verify.ts && tsx server/key-probes.verify.ts && tsx src/agent/settings/agent-settings.verify.ts && tsx src/editor/layouts.verify.ts && tsx src/audio/silence.verify.ts && tsx src/editor/silenceRebuild.verify.ts && tsx src/color/scopes.verify.ts && tsx src/audio/beats.verify.ts && tsx src/editor/keyframes.verify.ts && tsx src/editor/camSwitch.verify.ts && tsx src/export/fcpxml.verify.ts && tsx src/agent/timelineDelta.verify.ts && tsx server/plugins/extract-frames.verify.ts && tsx src/agent/tools/undo-tools.verify.ts && tsx src/editor/clipFit.verify.ts && tsx src/editor/sourceLimit.verify.ts && tsx src/editor/rippleChain.verify.ts && tsx src/editor/snap.verify.ts && tsx src/transcript/cutPad.verify.ts && tsx src/agent/tools/track-gaps.verify.ts && tsx src/persist/uploadRefs.verify.ts && tsx server/plugins/result-download.verify.ts && tsx src/agent/systemPromptOrder.verify.ts && tsx src/review/reviewModel.verify.ts && tsx src/editor/historyGesture.verify.ts && tsx src/agent/abortedTurn.verify.ts && tsx src/agent/changeLog.verify.ts && tsx server/plugins/reference-mime.verify.ts && tsx src/components/inspector/inspector-controls.verify.ts && tsx --tsconfig tsconfig.app.json src/components/timeline/timelineFitRange.verify.ts && tsx --tsconfig tsconfig.app.json src/components/timeline/timeline-clip-duration.verify.ts && tsx --tsconfig tsconfig.app.json src/components/timeline/trackOverlap.verify.ts && node src/components/timeline/playhead-watchdog.verify.mjs && tsx --tsconfig tsconfig.app.json src/components/timeline/timelineSeek.verify.ts && tsx --tsconfig tsconfig.app.json src/components/timeline/trackDelete.verify.ts && tsx --tsconfig tsconfig.app.json src/components/timeline/track-mute-visual.verify.ts && npm run test:contrib-timeline-preview", + "test:serial": "node remotion/performance.verify.mjs && tsx src/transcript/local-asr-warmup.verify.ts && tsx src/transcript/assemblyai.verify.ts && tsx src/transcript/assemblyai-resume.verify.ts && tsx src/transcript/transcribe-jobs.verify.ts && tsx src/persist/mediaIntelligenceStores.verify.ts && tsx src/editor/rateStretch.verify.ts && tsx server/media-acceleration.verify.ts && tsx server/plugins/scene-detection.verify.ts && tsx server/plugins/stock.verify.ts && tsx src/agent/proposal.verify.ts && tsx src/agent/external-edit-session.verify.ts && tsx src/components/chat/widget-parse.verify.ts && tsx src/agent/tools/generation-tool-input.verify.ts && tsx server/plugins/image.verify.ts && tsx server/plugins/video.verify.ts && tsx server/plugins/music.verify.ts && tsx server/plugins/music-sonilo.verify.ts && tsx server/plugins/music-media.verify.ts && tsx server/plugins/sound.verify.ts && tsx server/plugins/voice.verify.ts && tsx src/agent/tools/followup-tools.verify.ts && tsx src/agent/tools/clean-script-rules.verify.ts && tsx src/agent/tools/mg-export-params.verify.ts && tsx src/agent/tools/scene-detection-tools.verify.ts && tsx src/agent/tools/isolate-voice-tools.verify.ts && tsx src/agent/tools/template-tools.verify.ts && tsx src/agent/tools/script-tools.verify.ts && tsx src/agent/tools/timeline-target-tools.verify.ts && tsx src/agent/tools/track-tools.verify.ts && tsx server/plugins/generation-jobs.verify.ts && tsx server/plugins/export-runtime.verify.ts && tsx server/plugins/export-params.verify.ts && tsx src/export/browserExport.verify.ts && tsx src/agent/ai-sdk.verify.ts && tsx server/llm-proxy.verify.ts && tsx server/external-agent/broker.verify.ts && tsx server/plugins/project-store.verify.ts && tsx server/plugins/extension-store.verify.ts && tsx server/plugins/normalize-media.verify.ts && tsx src/persist/projectStore.verify.ts && tsx src/persist/design-style-metadata.verify.ts && tsx src/persist/designStyleTransfer.verify.ts && tsx src/captions/manualCaptions.verify.ts && tsx src/captions/captionPreviewTarget.verify.ts && tsx src/agent/tools/captions-track-order.verify.ts && tsx src/skins.verify.ts && tsx src/theme-tokenization.verify.ts && tsx server/mobile-upload-service.verify.ts && tsx src/media/mobileUploadApi.verify.ts && tsx src/tracking/tracking.verify.ts && tsx src/media/semantic-search/semanticSearch.verify.ts && tsx src/color/autoGrade.verify.ts && tsx server/plugins/auto-grade.verify.ts && tsx src/transcript/phrases.verify.ts && tsx src/agent/tools/transcript-read.verify.ts && tsx src/export/quality.verify.ts && tsx src/export/autoQa.verify.ts && tsx server/plugins/export-qa.verify.ts && tsx src/agent/tools/export-qa-tools.verify.ts && tsx src/persist/migrations/migrations.verify.ts && tsx server/keystore.verify.ts && tsx src/components/timeline/transition-context-menu.verify.ts && tsx shared/model-capabilities.verify.ts && tsx desktop/agent-path-import.verify.ts && tsx desktop/agent-path-import.e2e.verify.ts && tsx src/agent/tools/agent-path-import-tools.verify.ts && tsx server/key-probes.verify.ts && tsx src/agent/settings/agent-settings.verify.ts && tsx src/editor/layouts.verify.ts && tsx src/audio/silence.verify.ts && tsx src/editor/silenceRebuild.verify.ts && tsx src/color/scopes.verify.ts && tsx src/audio/beats.verify.ts && tsx src/editor/keyframes.verify.ts && tsx src/editor/camSwitch.verify.ts && tsx src/export/fcpxml.verify.ts && tsx src/agent/timelineDelta.verify.ts && tsx server/plugins/extract-frames.verify.ts && tsx src/agent/tools/undo-tools.verify.ts && tsx src/editor/clipFit.verify.ts && tsx src/editor/sourceLimit.verify.ts && tsx src/editor/rippleChain.verify.ts && tsx src/editor/snap.verify.ts && tsx src/transcript/cutPad.verify.ts && tsx src/agent/tools/track-gaps.verify.ts && tsx src/persist/uploadRefs.verify.ts && tsx server/plugins/result-download.verify.ts && tsx src/agent/systemPromptOrder.verify.ts && tsx src/review/reviewModel.verify.ts && tsx src/editor/historyGesture.verify.ts && tsx src/agent/abortedTurn.verify.ts && tsx src/agent/changeLog.verify.ts && tsx server/plugins/reference-mime.verify.ts && tsx src/components/inspector/inspector-controls.verify.ts && tsx --tsconfig tsconfig.app.json src/components/timeline/timelineFitRange.verify.ts && tsx --tsconfig tsconfig.app.json src/components/timeline/timeline-clip-duration.verify.ts && tsx --tsconfig tsconfig.app.json src/components/timeline/trackOverlap.verify.ts && node src/components/timeline/playhead-watchdog.verify.mjs && tsx --tsconfig tsconfig.app.json src/components/timeline/timelineSeek.verify.ts && tsx --tsconfig tsconfig.app.json src/components/timeline/trackDelete.verify.ts && tsx --tsconfig tsconfig.app.json src/components/timeline/track-mute-visual.verify.ts && npm run test:contrib-timeline-preview", "posttest": "npm run verify:background-fill && npm run verify:speech-providers && npm run verify:desktop-worker-lifecycle && npm run verify:server-runs && npm run verify:extract-audio", "verify:server-runs": "tsx server/agent-runs/model.verify.ts && tsx server/agent-runs/tool-catalog-generation.verify.ts && tsx server/agent-runs/codex-turn.verify.ts && tsx server/agent-runs/executor.verify.ts && tsx server/agent-runs/tool-policy.verify.ts && tsx server/agent-runs/store.verify.ts && tsx server/agent-runs/store-approval.verify.ts && tsx server/agent-runs/store-capability.verify.ts && tsx server/agent-runs/routes.verify.ts && tsx src/agent/useServerRun.verify.ts && tsx src/agent/serverRunFetchEventStream.verify.ts && tsx src/agent/serverRunSessionStorage.verify.ts && tsx src/agent/serverRunModelHistory.verify.ts && tsx src/agent/serverRunOwnership.verify.ts && tsx src/agent/serverRunProtocol.verify.ts && tsx src/agent/serverRunToolExecutor.verify.ts && tsx src/agent/serverRunToolRecovery.verify.ts && tsx src/agent/useAgentHistoryActions.verify.ts && tsx src/components/chat/serverRunInspector.verify.ts", "verify:background-fill": "tsx src/editor/backgroundFill.verify.ts && tsx --tsconfig tsconfig.app.json src/components/preview/previewTransform.verify.ts && tsx --tsconfig tsconfig.app.json src/components/inspector/background-fill-control.verify.tsx && tsx src/agent/tools/edit-item-background-fill.verify.ts && tsx src/export/fcpxml-background-fill.verify.ts", diff --git a/server/key-probes.ts b/server/key-probes.ts index 4c64a716..c7b39b12 100644 --- a/server/key-probes.ts +++ b/server/key-probes.ts @@ -328,6 +328,14 @@ export const PROBES: Record = { }), }, 'music/minimax': minimaxProbe, + // Read-only account endpoint; fake key → 401. The same key also serves + // /generate/sound (video-to-SFX), so one probe covers both capabilities. + 'music/sonilo': { + needs: [['SONILO_API_KEY']], + run: (get) => fetch(`${base(get, 'SONILO_BASE_URL', 'https://api.sonilo.com')}/v1/account/services`, { + signal: t(), headers: bearer(get('SONILO_API_KEY')), + }), + }, // /v1/search has been opened to anonymous users (the measured number is 200 without key), and the key; collections cannot be detected // It is the account binding endpoint, and the fake key is stable 401. 'stock/pexels': { diff --git a/server/key-probes.verify.ts b/server/key-probes.verify.ts index 120f2c2c..d163abf1 100644 --- a/server/key-probes.verify.ts +++ b/server/key-probes.verify.ts @@ -29,7 +29,7 @@ const EXPECTED_PAGES = [ 'voice/elevenlabs', 'voice/openai', 'voice/gemini', 'voice/mistral', 'voice/cartesia', 'voice/doubao', 'voice/minimax', 'voice/inworld', 'voice/fishaudio', 'voice/speechify', 'video/seedance', 'video/kling', 'video/hailuo', 'video/byteplus', - 'music/mureka', 'music/minimax', + 'music/mureka', 'music/minimax', 'music/sonilo', 'stock/pexels', 'stock/pixabay', 'stock/unsplash', 'stock/freesound', 'transcription/assemblyai', 'transcription/openai', 'transcription/mistral', 'transcription/deepgram', 'transcription/groq', 'transcription/elevenlabs', 'transcription/cartesia', diff --git a/server/keystore.ts b/server/keystore.ts index de2596c7..f45eb1a3 100644 --- a/server/keystore.ts +++ b/server/keystore.ts @@ -112,6 +112,8 @@ export const KEY_NAMES = [ "MUREKA_BASE_URL", "MINIMAX_API_KEY", "MINIMAX_BASE_URL", + "SONILO_API_KEY", + "SONILO_BASE_URL", "PEXELS_API_KEY", "PIXABAY_API_KEY", "UNSPLASH_ACCESS_KEY", @@ -373,8 +375,8 @@ export function computeCaps(): Caps { (getKey("PREFERRED_VOICE_VENDOR") === "cartesia" && has("CARTESIA_API_KEY")), video: has("SEEDANCE_API_KEY") || has("KLING_API_KEY") || has("MINIMAX_API_KEY") || has("BYTEPLUS_API_KEY"), - music: has("MUREKA_API_KEY") || has("MINIMAX_API_KEY"), - sound: has("ELEVENLABS_API_KEY"), + music: has("MUREKA_API_KEY") || has("MINIMAX_API_KEY") || has("SONILO_API_KEY"), + sound: has("ELEVENLABS_API_KEY") || has("SONILO_API_KEY"), stock: has("PEXELS_API_KEY") || has("PIXABAY_API_KEY") || diff --git a/server/plugins/generation-job-types.ts b/server/plugins/generation-job-types.ts index 340f6df8..5578c21f 100644 --- a/server/plugins/generation-job-types.ts +++ b/server/plugins/generation-job-types.ts @@ -15,6 +15,9 @@ export interface GenerationResult { fps?: number; /** Offset of a ranged export within the source timeline. */ sourceStartSeconds?: number; + /** Provider license record id for the generated track (also archived as a + * `.license.json` sidecar beside the audio file). */ + licenseId?: string; sizeBytes?: number; codec?: string; encoder?: H264EncoderProfile; diff --git a/server/plugins/index.ts b/server/plugins/index.ts index e3f41e15..97aececf 100644 --- a/server/plugins/index.ts +++ b/server/plugins/index.ts @@ -211,6 +211,12 @@ export function serverPlugins(options: { projectStoreHttp?: boolean } = {}): Plu get model() { return getKey("ELEVENLABS_SOUND_MODEL") || "eleven_text_to_sound_v2"; }, + get soniloBaseUrl() { + return getKey("SONILO_BASE_URL") || "https://api.sonilo.com"; + }, + get soniloApiKey() { + return getKey("SONILO_API_KEY"); + }, }), musicGenerationPlugin({ get baseUrl() { @@ -231,6 +237,12 @@ export function serverPlugins(options: { projectStoreHttp?: boolean } = {}): Plu get minimaxModel() { return getKey("MINIMAX_MUSIC_MODEL") || "music-2.6"; }, + get soniloBaseUrl() { + return getKey("SONILO_BASE_URL") || "https://api.sonilo.com"; + }, + get soniloApiKey() { + return getKey("SONILO_API_KEY"); + }, }), videoGenerationPlugin({ get seedanceBaseUrl() { diff --git a/server/plugins/music-sonilo.ts b/server/plugins/music-sonilo.ts new file mode 100644 index 00000000..755901e5 --- /dev/null +++ b/server/plugins/music-sonilo.ts @@ -0,0 +1,52 @@ +import type { GenerationResult } from './generation-jobs.ts'; +import { fetchGeneratedResult } from './result-download.ts'; +import { + SONILO_MUSIC_ENDPOINT, + SONILO_MUSIC_MAX_VIDEO_SECONDS, + assertSoniloVideoDuration, + awaitSoniloTracks, + saveSoniloAudioResponse, + submitSoniloVideoTask, + writeSoniloLicenseSidecar, + type SoniloAudioTrack, +} from './sonilo-media.ts'; +import type { MusicOptions, ValidMusicRequest } from './music-types.ts'; + +/** Video-to-music from the finished cut. One durable result; the optional + * single style prompt is passed through, and the task id is registered before + * polling so a restarted server resumes instead of resubmitting. */ +export async function generateSoniloMusic( + options: MusicOptions, + input: ValidMusicRequest, + onTaskAccepted: (taskId: string) => Promise, + existingTaskId?: string, +): Promise { + const baseUrl = options.soniloBaseUrl.replace(/\/$/, ''); + let taskId = existingTaskId; + if (!taskId) { + if (!input.sourceAssetPath) throw new Error('Sonilo v2m requires a project video sourceAssetId'); + await assertSoniloVideoDuration(input.sourceAssetPath, SONILO_MUSIC_MAX_VIDEO_SECONDS, 'music'); + taskId = await submitSoniloVideoTask( + baseUrl, + options.soniloApiKey, + SONILO_MUSIC_ENDPOINT, + input.sourceAssetPath, + input.prompt || undefined, + ); + await onTaskAccepted(taskId); + } + return awaitSoniloTracks(baseUrl, options.soniloApiKey, taskId); +} + +/** Download one result track from its presigned URL (no auth header) and + * archive the per-track license id as a sidecar record. */ +export async function soniloMusicResult( + jobId: string, + input: ValidMusicRequest, + url: string, + licenseId?: string, +): Promise { + const saved = await saveSoniloAudioResponse(await fetchGeneratedResult(url, 'audio'), url); + if (licenseId) await writeSoniloLicenseSidecar(saved.path, licenseId); + return { assetId: jobId, kind: 'audio', name: input.name, licenseId, ...saved }; +} diff --git a/server/plugins/music-sonilo.verify.ts b/server/plugins/music-sonilo.verify.ts new file mode 100644 index 00000000..77914754 --- /dev/null +++ b/server/plugins/music-sonilo.verify.ts @@ -0,0 +1,120 @@ +import assert from 'node:assert/strict'; +import { expectedMusicResultCount, validateMusicRequest } from './music.ts'; +import { + SONILO_MUSIC_MAX_VIDEO_SECONDS, + SONILO_SFX_MAX_VIDEO_SECONDS, + pickSoniloTracks, + soniloTaskId, +} from './sonilo-media.ts'; + +// ── request validation ── +const promptless = validateMusicRequest({ + provider: 'sonilo', sourceAssetPath: '/media/uploads/cut.mp4', sourceAssetKind: 'video', +}); +assert.equal(promptless.provider, 'sonilo'); +assert.equal(promptless.mode, 'v2m', 'sonilo infers v2m without an explicit mode'); +assert.equal(promptless.prompt, '', 'sonilo v2m works promptless'); +assert.equal(promptless.count, 1); +assert.equal(promptless.isInstrumental, true); +assert.equal(expectedMusicResultCount(promptless), 1, 'sonilo returns exactly one result'); + +const styled = validateMusicRequest({ + provider: 'sonilo', mode: 'v2m', sourceAssetPath: '/media/uploads/cut.mp4', sourceAssetKind: 'video', + prompt: 'warm indie folk, no drums', name: 'Score · final cut', +}); +assert.equal(styled.prompt, 'warm indie folk, no drums'); +assert.equal(styled.name, 'Score · final cut'); + +assert.throws(() => validateMusicRequest({ provider: 'sonilo' }), /video sourceAssetId/); +assert.throws( + () => validateMusicRequest({ provider: 'sonilo', sourceAssetPath: '/media/uploads/a.png', sourceAssetKind: 'image' }), + /video sourceAssetId/, +); +assert.throws( + () => validateMusicRequest({ provider: 'sonilo', mode: 'soundtrack', sourceAssetPath: '/media/uploads/cut.mp4', sourceAssetKind: 'video' }), + /mode must be v2m/, +); +assert.throws( + () => validateMusicRequest({ + provider: 'sonilo', sourceAssetPath: '/media/uploads/cut.mp4', sourceAssetKind: 'video', prompt: 'x'.repeat(501), + }), + /at most 500/, +); +assert.throws( + () => validateMusicRequest({ + provider: 'sonilo', sourceAssetPath: '/media/uploads/cut.mp4', sourceAssetKind: 'video', lyrics: 'hello', + }), + /does not take lyrics/, +); +assert.throws( + () => validateMusicRequest({ + provider: 'sonilo', sourceAssetPath: '/media/uploads/cut.mp4', sourceAssetKind: 'video', styles: ['pop'], + }), + /mureka provider only/, +); +assert.throws( + () => validateMusicRequest({ + provider: 'sonilo', sourceAssetPath: '/media/uploads/cut.mp4', sourceAssetKind: 'video', bitrate: 128_000, + }), + /not supported by sonilo/, +); +assert.throws( + () => validateMusicRequest({ + provider: 'sonilo', sourceAssetPath: '/media/uploads/cut.mp4', sourceAssetKind: 'video', count: 2, + }), + /count must be 1/, +); +assert.throws( + () => validateMusicRequest({ + provider: 'sonilo', sourceAssetPath: '/media/uploads/cut.mp4', sourceAssetKind: 'video', audioFormat: 'mp3', + }), + /not configurable/, +); +// sonilo controls must not leak into the other providers +assert.throws(() => validateMusicRequest({ provider: 'minimax', prompt: 'lofi', mode: 'v2m' }), /t2m or cover/); +assert.throws(() => validateMusicRequest({ provider: 'mureka', mode: 'v2m', prompt: 'lofi' }), /mureka mode must be/); +assert.throws(() => validateMusicRequest({ provider: 'suno', prompt: 'lofi' }), /mureka, minimax, or sonilo/); + +// ── task shapes ── +assert.equal(soniloTaskId({ task_id: 'task-1' }), 'task-1'); +assert.equal(soniloTaskId({ id: 'task-2' }), 'task-2'); +assert.equal(soniloTaskId({}), undefined); + +const singleTrack = pickSoniloTracks({ + status: 'succeeded', + result: { audio: { url: 'https://cdn.example/a.m4a', license_id: 'lic-a' } }, +}); +assert.deepEqual(singleTrack, [{ name: 'audio', url: 'https://cdn.example/a.m4a', licenseId: 'lic-a' }]); + +const multiTrack = pickSoniloTracks({ + status: 'succeeded', + tracks: [ + { name: 'main', url: 'https://cdn.example/main.m4a', license_id: 'lic-main' }, + { audio_url: 'https://cdn.example/alt.m4a' }, + { name: 'broken' }, + ], +}); +assert.equal(multiTrack.length, 2, 'entries without a URL are skipped'); +assert.equal(multiTrack[0].name, 'main', 'primary track stays first'); +assert.equal(multiTrack[0].licenseId, 'lic-main'); +assert.equal(multiTrack[1].url, 'https://cdn.example/alt.m4a'); +assert.equal(multiTrack[1].licenseId, undefined, 'missing license_id stays undefined'); + +const keyedResult = pickSoniloTracks({ + status: 'succeeded', + result: { + audio: { url: 'https://cdn.example/a.m4a', license_id: 'lic-a' }, + stems: [{ url: 'https://cdn.example/b.m4a', license_id: 'lic-b' }], + meta: 'not-a-track', + }, +}); +assert.equal(keyedResult.length, 2); +assert.deepEqual(keyedResult[1], { name: 'stems_1', url: 'https://cdn.example/b.m4a', licenseId: 'lic-b' }); + +assert.deepEqual(pickSoniloTracks({ status: 'succeeded' }), [], 'empty result yields no tracks'); + +// ── documented caps ── +assert.equal(SONILO_MUSIC_MAX_VIDEO_SECONDS, 360, 'music cap is a 6-minute video'); +assert.equal(SONILO_SFX_MAX_VIDEO_SECONDS, 180, 'SFX cap is a 3-minute video'); + +console.log('music-sonilo.check: ok (v2m validation + task track/license extraction)'); diff --git a/server/plugins/music-types.ts b/server/plugins/music-types.ts index cd37cbe1..ee5149d6 100644 --- a/server/plugins/music-types.ts +++ b/server/plugins/music-types.ts @@ -1,5 +1,5 @@ -export type MusicProvider = 'mureka' | 'minimax'; -export type MusicMode = 'instrumental' | 'song' | 'prompt-song' | 'soundtrack' | 'track' | 't2m' | 'cover'; +export type MusicProvider = 'mureka' | 'minimax' | 'sonilo'; +export type MusicMode = 'instrumental' | 'song' | 'prompt-song' | 'soundtrack' | 'track' | 't2m' | 'cover' | 'v2m'; export type MusicAudioFormat = 'mp3' | 'wav' | 'pcm' | 'flac'; export type MurekaStyle = 'pop' | 'rock' | 'jazz' | 'r&b' | 'edm' | 'ambient' | 'folk' | 'latin' | 'k-pop' | 'j-pop' | 'house' | 'gospel' | 'lo-fi'; @@ -13,6 +13,8 @@ export interface MusicOptions { minimaxBaseUrl: string; minimaxApiKey: string; minimaxModel: string; + soniloBaseUrl: string; + soniloApiKey: string; } export interface MusicRequest { diff --git a/server/plugins/music-validation.ts b/server/plugins/music-validation.ts index 375593d8..97c06d5e 100644 --- a/server/plugins/music-validation.ts +++ b/server/plugins/music-validation.ts @@ -57,6 +57,27 @@ function rejectMurekaOnly(input: MusicRequest): void { if (values.some((value) => value !== undefined)) throw new Error('Mureka generation controls are supported by the mureka provider only'); } +// Sonilo v2m scores the finished cut: the video is the input, the prompt is a +// single optional style hint (works without one). No lyrics/stems/segment +// controls exist, output is always one m4a track, and /v1 picks the model. +function validateSonilo(input: MusicRequest, mode: MusicMode, prompt: string, lyrics?: string): void { + if (mode !== 'v2m') throw new Error('sonilo mode must be v2m'); + if (!input.sourceAssetPath || input.sourceAssetKind !== 'video') { + throw new Error('sonilo v2m requires a project video sourceAssetId (the rendered cut)'); + } + if (prompt.length > 500) throw new Error('sonilo style prompt must be at most 500 characters'); + if (lyrics) throw new Error('sonilo v2m does not take lyrics; music is generated from the video'); + const murekaOnly = [input.styles, input.gender, input.referenceId, input.instrumentalId, input.vocalId, input.melodyId, + input.audioStartMs, input.audioEndMs, input.songId, input.trackType, input.generateStartMs, input.generateEndMs, + input.vocalGender, input.stream]; + if (murekaOnly.some((value) => value !== undefined)) throw new Error('Mureka generation controls are supported by the mureka provider only'); + const minimaxOnly = [input.referenceAudioPath, input.coverFeatureId, input.lyricsOptimizer, input.isInstrumental, + input.sampleRate, input.bitrate]; + if (minimaxOnly.some((value) => value !== undefined)) throw new Error('MiniMax-only controls are not supported by sonilo'); + if (input.count !== undefined && input.count !== 1) throw new Error('sonilo v2m returns exactly one result; count must be 1'); + if (input.audioFormat) throw new Error('sonilo output format is not configurable (m4a is returned)'); +} + function validateMinimax(input: MusicRequest, mode: MusicMode, prompt: string, lyrics?: string): void { if (mode !== 't2m' && mode !== 'cover') throw new Error('minimax mode must be t2m or cover'); rejectMurekaOnly(input); @@ -84,12 +105,15 @@ function validateMinimax(input: MusicRequest, mode: MusicMode, prompt: string, l export function validateMusicRequest(input: MusicRequest): ValidMusicRequest { const provider = String(input.provider ?? 'mureka'); - if (provider !== 'mureka' && provider !== 'minimax') throw new Error('provider must be mureka or minimax'); - const inferredMode: MusicMode = provider === 'mureka' ? 'instrumental' : input.referenceAudioPath || input.coverFeatureId ? 'cover' : 't2m'; + if (provider !== 'mureka' && provider !== 'minimax' && provider !== 'sonilo') throw new Error('provider must be mureka, minimax, or sonilo'); + const inferredMode: MusicMode = provider === 'mureka' ? 'instrumental' + : provider === 'sonilo' ? 'v2m' + : input.referenceAudioPath || input.coverFeatureId ? 'cover' : 't2m'; const mode = input.mode ?? inferredMode; const prompt = optionalText(input.prompt) ?? ''; const lyrics = optionalText(input.lyrics); - if (provider === 'mureka') { + if (provider === 'sonilo') validateSonilo(input, mode, prompt, lyrics); + else if (provider === 'mureka') { if (!MUREKA_MODES.has(mode)) throw new Error('mureka mode must be instrumental, song, prompt-song, soundtrack, or track'); if (input.referenceAudioPath || input.coverFeatureId || input.lyricsOptimizer !== undefined || input.isInstrumental !== undefined || input.sampleRate !== undefined || input.bitrate !== undefined) throw new Error('MiniMax-only controls are not supported by mureka'); @@ -103,7 +127,9 @@ export function validateMusicRequest(input: MusicRequest): ValidMusicRequest { ...input, provider, mode, prompt, lyrics, name, isInstrumental: provider === 'mureka' ? mode === 'instrumental' - : input.isInstrumental ?? (mode === 't2m' && !lyrics && !input.lyricsOptimizer), + : provider === 'sonilo' + ? true + : input.isInstrumental ?? (mode === 't2m' && !lyrics && !input.lyricsOptimizer), lyricsOptimizer: input.lyricsOptimizer === true, sampleRate: input.sampleRate ?? 44_100, bitrate: input.bitrate ?? 256_000, diff --git a/server/plugins/music.ts b/server/plugins/music.ts index b64d7bee..50a93691 100644 --- a/server/plugins/music.ts +++ b/server/plugins/music.ts @@ -14,6 +14,7 @@ import { import { saveAudioResponse } from './music-media.ts'; import { generateMureka, pickMurekaAudioUrl } from './music-mureka.ts'; import { isMinimaxCoverModel, minimaxMusicUrl } from './music-minimax.ts'; +import { generateSoniloMusic, soniloMusicResult } from './music-sonilo.ts'; import type { MusicOptions, MusicRequest, ValidMusicRequest } from './music-types.ts'; import { validateMusicRequest } from './music-validation.ts'; import { fetchGeneratedResult } from './result-download.ts'; @@ -72,27 +73,43 @@ async function runMusicOperation( const expectedResultCount = expectedMusicResultCount(input); const checkpoint = generationResultCheckpoint(storedResultUrls, expectedResultCount, providerTaskId); let urls = checkpoint.urls; + // Sonilo license ids ride alongside the single result URL; a resume from a + // stored URL checkpoint recovers the audio, the sidecar stays best-effort. + let soniloLicenseIds: (string | undefined)[] = []; if (!checkpoint.complete) { - const providerUrls = input.provider === 'minimax' - ? [await minimaxMusicUrl(options, input)] - : await generateMureka( + let providerUrls: string[]; + if (input.provider === 'minimax') providerUrls = [await minimaxMusicUrl(options, input)]; + else if (input.provider === 'sonilo') { + const tracks = await generateSoniloMusic( + options, + input, + (taskId) => registerProviderTask('sonilo', taskId), + providerTaskId, + ); + providerUrls = tracks.map((track) => track.url); + soniloLicenseIds = tracks.map((track) => track.licenseId); + } else { + providerUrls = await generateMureka( options, input, (taskId) => registerProviderTask('mureka', taskId), providerTaskId, ); + } urls = requireGenerationResultUrls(providerUrls, expectedResultCount); } urls = requireGenerationResultUrls(urls, expectedResultCount); const download = () => input.provider === 'minimax' ? minimaxResult(operationId, input, urls[0]) - : murekaResults(operationId, input, urls); + : input.provider === 'sonilo' + ? soniloMusicResult(operationId, input, urls[0], soniloLicenseIds[0]) + : murekaResults(operationId, input, urls); for (const [index, url] of urls.entries()) await registerDownload(url, download, index); return download(); } export function musicGenerationPlugin(options: MusicOptions): Plugin { - for (const provider of ['mureka', 'minimax'] as const) { + for (const provider of ['mureka', 'minimax', 'sonilo'] as const) { registerGenerationJobResumer('submit_music', provider, async ( snapshot: GenerationJobSnapshot, _update, @@ -122,7 +139,13 @@ export function musicGenerationPlugin(options: MusicOptions): Plugin { if (input.provider === 'mureka' && !options.apiKey) { throw new Error('Mureka is not configured. Set MUREKA_API_KEY in .env.local or 设置面板.'); } - const model = input.provider === 'minimax' ? options.minimaxModel : options.model; + if (input.provider === 'sonilo' && !options.soniloApiKey) { + throw new Error('Sonilo is not configured. Set SONILO_API_KEY in .env.local or 设置面板.'); + } + // Sonilo has no client-selected model: /v1 routes to the latest server-side. + const model = input.provider === 'minimax' ? options.minimaxModel + : input.provider === 'sonilo' ? 'v1' + : options.model; const submitArgs = Object.fromEntries(Object.entries(raw).filter(([key]) => key !== 'operationId')); const submission = await createGenerationJob( { kind: 'music', model, ...input }, diff --git a/server/plugins/sonilo-media.ts b/server/plugins/sonilo-media.ts new file mode 100644 index 00000000..1fb1fbcb --- /dev/null +++ b/server/plugins/sonilo-media.ts @@ -0,0 +1,235 @@ +import { proxyDispatcher } from '../outbound-proxy.ts'; +import { createWriteStream } from 'node:fs'; +import { spawn } from 'node:child_process'; +import { randomUUID } from 'node:crypto'; +import { mkdir, readFile, rename, rm, stat, writeFile } from 'node:fs/promises'; +import { extname, join } from 'node:path'; +import { Readable } from 'node:stream'; +import { pipeline } from 'node:stream/promises'; +import type { ReadableStream as WebReadableStream } from 'node:stream/web'; + +import { isSafeUploadName, resolveUploadFile, uploadDir } from '../media-dir.ts'; +// Proxy-aware fetch: attaches the configured outbound proxy (keystore +// PROXY_URL or HTTPS_PROXY/HTTP_PROXY env) via undici dispatcher. +type FetchInit = Parameters[1] & { dispatcher?: unknown }; +const fetchWithProxy = (url: RequestInfo | URL, init?: FetchInit): Promise => + fetch(url, { ...init, dispatcher: proxyDispatcher() } as RequestInit); + +// Sonilo scores the finished cut: /v1/video-to-music (≤6-minute video) and +// /v1/video-to-sfx (≤3-minute video). Both are async: multipart submit → +// poll GET /v1/tasks/{task_id} → presigned audio URL + per-track license_id. +// /v1 routes to the latest model server-side; no model id is sent. +export const SONILO_MUSIC_MAX_VIDEO_SECONDS = 360; +export const SONILO_SFX_MAX_VIDEO_SECONDS = 180; +export const SONILO_MUSIC_ENDPOINT = '/v1/video-to-music'; +export const SONILO_SFX_ENDPOINT = '/v1/video-to-sfx'; +// Partner-origin attribution header; the backend has no other way to tell +// which integration a request came from. +const SONILO_USER_AGENT = 'OpenChatCut'; + +const TERMINAL_FAILURES = new Set(['failed', 'timeouted', 'cancelled']); +const wait = (milliseconds: number) => new Promise((resolvePromise) => setTimeout(resolvePromise, milliseconds)); + +export interface SoniloTrack { + name?: string; + url?: string; + audio_url?: string; + license_id?: string; +} + +export interface SoniloTask { + task_id?: string; + id?: string; + status?: string; + error?: string; + failed_reason?: string; + result?: Record; + tracks?: SoniloTrack[]; +} + +export interface SoniloAudioTrack { + name: string; + url: string; + licenseId?: string; +} + +export async function soniloProviderError(response: Response): Promise { + const text = await response.text(); + try { + const data = JSON.parse(text) as { message?: string; detail?: string; error?: { message?: string } | string }; + const error = typeof data.error === 'string' ? data.error : data.error?.message; + return error ?? data.message ?? data.detail ?? `Sonilo request failed (${response.status})`; + } catch { + return text.slice(0, 300) || `Sonilo request failed (${response.status})`; + } +} + +export function soniloTaskId(task: SoniloTask): string | undefined { + const id = task.task_id ?? task.id; + return id ? String(id) : undefined; +} + +/** Flatten every audio-carrying object in the task result, primary track first. + * Tolerates both a `tracks` array and single/keyed objects under `result`; + * `license_id` is attached when the provider returns one. */ +export function pickSoniloTracks(task: SoniloTask): SoniloAudioTrack[] { + const tracks: SoniloAudioTrack[] = []; + const push = (name: string, candidate: unknown): void => { + if (!candidate || typeof candidate !== 'object') return; + const entry = candidate as SoniloTrack; + const url = entry.url ?? entry.audio_url; + if (!url) return; + tracks.push({ name: entry.name ?? name, url, licenseId: entry.license_id }); + }; + for (const [index, entry] of (task.tracks ?? []).entries()) push(`track_${index + 1}`, entry); + const result = task.result ?? {}; + for (const [key, value] of Object.entries(result)) { + if (Array.isArray(value)) for (const [index, entry] of value.entries()) push(`${key}_${index + 1}`, entry); + else push(key, value); + } + return tracks; +} + +function localVideoUpload(uploadPath: string): { file: string; name: string } { + const clean = uploadPath.split(/[?#]/, 1)[0]; + if (!clean.startsWith('/media/uploads/')) throw new Error('Sonilo source must be a project upload'); + const name = clean.slice('/media/uploads/'.length); + if (!isSafeUploadName(name)) throw new Error('invalid Sonilo source path'); + const file = resolveUploadFile(name); + if (!file) throw new Error(`Sonilo source not found: ${uploadPath}`); + return { file, name }; +} + +export function probeMediaDurationSeconds(file: string): Promise { + return new Promise((resolvePromise, reject) => { + const child = spawn('ffprobe', ['-v', 'error', '-show_entries', 'format=duration', '-of', 'default=nw=1:nk=1', file]); + let output = ''; + child.stdout.on('data', (data) => { output += String(data); }); + child.on('error', reject); + child.on('close', (code) => { + const duration = Number(output.trim()); + if (code === 0 && Number.isFinite(duration) && duration > 0) resolvePromise(duration); + else reject(new Error('unable to probe Sonilo source video')); + }); + }); +} + +/** Fail fast locally before uploading a cut the provider would reject. */ +export async function assertSoniloVideoDuration(uploadPath: string, maxSeconds: number, label: string): Promise { + const { file } = localVideoUpload(uploadPath); + const duration = await probeMediaDurationSeconds(file); + if (duration > maxSeconds) { + throw new Error(`Sonilo ${label} supports videos up to ${Math.round(maxSeconds / 60)} minutes; the source is ${Math.ceil(duration)}s`); + } +} + +function mimeFor(file: string): string { + const ext = extname(file).toLowerCase(); + if (ext === '.webm') return 'video/webm'; + if (ext === '.mov') return 'video/quicktime'; + return ext === '.mp4' ? 'video/mp4' : 'application/octet-stream'; +} + +/** Multipart submit of a project video upload; resolves to the accepted task id. */ +export async function submitSoniloVideoTask( + baseUrl: string, + apiKey: string, + endpoint: string, + uploadPath: string, + prompt?: string, +): Promise { + const { file, name } = localVideoUpload(uploadPath); + const bytes = await readFile(file); + if (!bytes.length) throw new Error('Sonilo source video is empty'); + const form = new FormData(); + form.append('file', new Blob([bytes], { type: mimeFor(file) }), name); + if (prompt) form.append('prompt', prompt); + const response = await fetchWithProxy(`${baseUrl.replace(/\/$/, '')}${endpoint}`, { + method: 'POST', + headers: { Authorization: `Bearer ${apiKey}`, 'User-Agent': SONILO_USER_AGENT }, + body: form, + }); + if (!response.ok) throw new Error(await soniloProviderError(response)); + const task = await response.json() as SoniloTask; + const id = soniloTaskId(task); + if (!id) throw new Error('Sonilo did not return a task id'); + return id; +} + +async function fetchSoniloTask(baseUrl: string, apiKey: string, taskId: string): Promise { + const response = await fetchWithProxy(`${baseUrl.replace(/\/$/, '')}/v1/tasks/${encodeURIComponent(taskId)}`, { + headers: { Authorization: `Bearer ${apiKey}`, 'User-Agent': SONILO_USER_AGENT }, + }); + // 404 on the poll is the documented fail-fast signal (task not found/expired), + // not a transient error worth retrying. + if (response.status === 404) throw new Error(`Sonilo task not found or expired: ${taskId}`); + if (!response.ok) throw new Error(await soniloProviderError(response)); + return response.json() as Promise; +} + +/** Poll the task to a terminal state and return its audio tracks (primary first). */ +export async function awaitSoniloTracks(baseUrl: string, apiKey: string, taskId: string): Promise { + const deadline = Date.now() + 10 * 60_000; + while (Date.now() < deadline) { + const task = await fetchSoniloTask(baseUrl, apiKey, taskId); + if (task.status === 'succeeded') { + const tracks = pickSoniloTracks(task); + if (!tracks.length) throw new Error('Sonilo succeeded without an audio track'); + return tracks; + } + if (task.status && TERMINAL_FAILURES.has(task.status)) { + throw new Error(task.failed_reason || task.error || `Sonilo generation ${task.status}`); + } + await wait(2_000); + } + throw new Error('Sonilo generation timed out'); +} + +function audioExtFor(url: string, contentType: string | null): string { + const clean = url.split(/[?#]/, 1)[0]; + const ext = extname(clean).replace('.', '').toLowerCase(); + if (['m4a', 'mp3', 'wav', 'flac', 'aac', 'ogg'].includes(ext)) return ext; + if (contentType?.includes('mpeg')) return 'mp3'; + if (contentType?.includes('wav')) return 'wav'; + return 'm4a'; +} + +/** Stream a (presigned — never send auth headers to it) result URL into the + * uploads dir. Sonilo outputs .m4a; the extension follows the URL/content type. */ +export async function saveSoniloAudioResponse( + response: Response, + sourceUrl: string, +): Promise<{ path: string; durationSeconds: number }> { + if (!response.ok) throw new Error(await soniloProviderError(response)); + if (!response.body) throw new Error('Sonilo returned empty audio'); + const ext = audioExtFor(sourceUrl, response.headers.get('content-type')); + const dir = uploadDir(); + await mkdir(dir, { recursive: true }); + const filename = `${randomUUID()}.${ext}`; + const file = join(dir, filename); + const partial = join(dir, `.${filename}.part`); + try { + await pipeline( + Readable.fromWeb(response.body as WebReadableStream), + createWriteStream(partial, { flags: 'wx' }), + ); + if (!(await stat(partial)).size) throw new Error('Sonilo returned empty audio'); + const durationSeconds = await probeMediaDurationSeconds(partial); + await rename(partial, file); + return { path: `/media/uploads/${filename}`, durationSeconds }; + } catch (error) { + await rm(partial, { force: true }).catch(() => undefined); + throw error; + } +} + +/** Durable per-track license record beside the audio file — an auditable + * artifact users can keep for commercial-use review. Best-effort: a sidecar + * failure never fails the generation itself. */ +export async function writeSoniloLicenseSidecar(audioPath: string, licenseId: string): Promise { + const name = audioPath.split('/').pop(); + if (!name || !isSafeUploadName(name)) return; + const sidecar = join(uploadDir(), `${name}.license.json`); + const record = { provider: 'sonilo', licenseId, audioPath, savedAt: new Date().toISOString() }; + await writeFile(sidecar, `${JSON.stringify(record, null, 2)}\n`).catch(() => undefined); +} diff --git a/server/plugins/sound.ts b/server/plugins/sound.ts index 109617c5..375d059a 100644 --- a/server/plugins/sound.ts +++ b/server/plugins/sound.ts @@ -7,6 +7,16 @@ import type { IncomingMessage, ServerResponse } from 'node:http'; import type { Plugin } from 'vite'; import { uploadDir } from '../media-dir.ts'; +import { + SONILO_SFX_ENDPOINT, + SONILO_SFX_MAX_VIDEO_SECONDS, + assertSoniloVideoDuration, + awaitSoniloTracks, + saveSoniloAudioResponse, + submitSoniloVideoTask, + writeSoniloLicenseSidecar, +} from './sonilo-media.ts'; +import { fetchGeneratedResult } from './result-download.ts'; // Proxy-aware fetch: attaches the configured outbound proxy (keystore // PROXY_URL or HTTPS_PROXY/HTTP_PROXY env) via undici dispatcher. type FetchInit = Parameters[1] & { dispatcher?: unknown }; @@ -22,26 +32,35 @@ const OUTPUT_FORMATS = new Set([ 'opus_48000_32', 'opus_48000_64', 'opus_48000_96', 'opus_48000_128', 'opus_48000_192', ]); +type SoundProvider = 'elevenlabs' | 'sonilo'; + interface SoundOptions { baseUrl: string; apiKey: string; model: string; + soniloBaseUrl: string; + soniloApiKey: string; } interface SoundRequest { + provider?: string; prompt?: string; durationSeconds?: number; promptInfluence?: number; loop?: boolean; outputFormat?: string; + sourceAssetPath?: string; + sourceAssetKind?: string; } export interface ValidSoundRequest { + provider: SoundProvider; prompt: string; durationSeconds?: number; promptInfluence: number; loop: boolean; outputFormat: string; + sourceAssetPath?: string; } async function readJson(req: IncomingMessage): Promise { @@ -74,19 +93,36 @@ async function providerError(response: Response): Promise { /** Pure validation — exported for unit checks. */ export function validateSoundRequest(input: SoundRequest): ValidSoundRequest { + const provider = String(input.provider ?? 'elevenlabs'); + if (provider !== 'elevenlabs' && provider !== 'sonilo') throw new Error('sound provider must be elevenlabs or sonilo'); const prompt = String(input.prompt ?? '').trim(); + if (provider === 'sonilo') { + // Sonilo video-to-SFX reads the cut itself — no prompt, no ElevenLabs + // synthesis controls; the source video is the whole request. + if (!input.sourceAssetPath || input.sourceAssetKind !== 'video') { + throw new Error('sonilo sound requires a project video sourceAssetId (the rendered cut)'); + } + if (prompt) throw new Error('sonilo sound is generated from the video; prompt is not supported'); + if ([input.durationSeconds, input.promptInfluence, input.loop, input.outputFormat].some((value) => value !== undefined)) { + throw new Error('ElevenLabs sound controls are not supported by sonilo'); + } + return { provider, prompt: '', promptInfluence: 0.3, loop: false, outputFormat: 'mp3_44100_128', sourceAssetPath: input.sourceAssetPath }; + } const durationSeconds = input.durationSeconds; const promptInfluence = input.promptInfluence ?? 0.3; const loop = input.loop ?? false; const outputFormat = String(input.outputFormat ?? 'mp3_44100_128'); if (!prompt) throw new Error('prompt is required'); + if (input.sourceAssetPath !== undefined || input.sourceAssetKind !== undefined) { + throw new Error('sourceAssetId is supported by the sonilo provider only'); + } if (durationSeconds != null && (!Number.isFinite(durationSeconds) || durationSeconds < 0.5 || durationSeconds > 30)) { throw new Error('durationSeconds must be between 0.5 and 30'); } if (!Number.isFinite(promptInfluence) || promptInfluence < 0 || promptInfluence > 1) throw new Error('promptInfluence must be between 0 and 1'); if (typeof loop !== 'boolean') throw new Error('loop must be a boolean'); if (!OUTPUT_FORMATS.has(outputFormat)) throw new Error(`unsupported ElevenLabs outputFormat ${outputFormat}`); - return { prompt, durationSeconds, promptInfluence, loop, outputFormat }; + return { provider, prompt, durationSeconds, promptInfluence, loop, outputFormat }; } const validate = validateSoundRequest; @@ -148,6 +184,26 @@ async function saveAudio(bytes: Buffer, outputFormat: string): Promise<{ path: s return { path: `/media/uploads/${filename}`, durationSeconds: await probeDuration(file) }; } +// Sonilo video-to-SFX: async on the provider side (submit → poll), consumed +// here within the synchronous /generate/sound contract — SFX for a ≤3-minute +// cut completes well inside the request window, matching the ElevenLabs path. +async function generateSoniloSound( + options: SoundOptions, + input: ValidSoundRequest, +): Promise<{ path: string; durationSeconds: number; licenseId?: string }> { + if (!options.soniloApiKey) throw new Error('Sonilo is not configured. Set SONILO_API_KEY in .env.local or 设置面板.'); + const sourcePath = input.sourceAssetPath!; + await assertSoniloVideoDuration(sourcePath, SONILO_SFX_MAX_VIDEO_SECONDS, 'SFX'); + const baseUrl = options.soniloBaseUrl.replace(/\/$/, ''); + const taskId = await submitSoniloVideoTask(baseUrl, options.soniloApiKey, SONILO_SFX_ENDPOINT, sourcePath); + const [primary] = await awaitSoniloTracks(baseUrl, options.soniloApiKey, taskId); + if (!primary) throw new Error('Sonilo returned no SFX track'); + // Presigned result URL — downloaded without auth headers by design. + const saved = await saveSoniloAudioResponse(await fetchGeneratedResult(primary.url, 'audio'), primary.url); + if (primary.licenseId) await writeSoniloLicenseSidecar(saved.path, primary.licenseId); + return { ...saved, licenseId: primary.licenseId }; +} + export function soundGenerationPlugin(options: SoundOptions): Plugin { return { name: 'openchatcut-sound-generation', @@ -155,8 +211,12 @@ export function soundGenerationPlugin(options: SoundOptions): Plugin { server.middlewares.use('/generate/sound', async (req, res) => { if (req.method !== 'POST') { sendJson(res, 405, { error: 'method not allowed — use POST' }); return; } try { - if (!options.apiKey) throw new Error('Sound generation is not configured. Set ELEVENLABS_API_KEY in .env.local.'); const input = validate(await readJson(req)); + if (input.provider === 'sonilo') { + sendJson(res, 200, await generateSoniloSound(options, input)); + return; + } + if (!options.apiKey) throw new Error('Sound generation is not configured. Set ELEVENLABS_API_KEY in .env.local.'); if (input.loop && options.model !== 'eleven_text_to_sound_v2') { throw new Error('loop requires ELEVENLABS_SOUND_MODEL eleven_text_to_sound_v2'); } diff --git a/server/plugins/sound.verify.ts b/server/plugins/sound.verify.ts index cff4c588..34826f3b 100644 --- a/server/plugins/sound.verify.ts +++ b/server/plugins/sound.verify.ts @@ -16,4 +16,29 @@ assert.throws(() => validateSoundRequest({ prompt: 'x', durationSeconds: 0.1 }), assert.throws(() => validateSoundRequest({ prompt: 'x', promptInfluence: 2 }), /promptInfluence must be between/); assert.throws(() => validateSoundRequest({ prompt: 'x', outputFormat: 'wav' }), /unsupported ElevenLabs outputFormat/); -console.log('sound.check: ok (elevenlabs official sound parameters)'); +// sonilo: SFX from the cut — video source in, no prompt, no synthesis controls +assert.equal(ok.provider, 'elevenlabs', 'provider defaults to elevenlabs'); +const sonilo = validateSoundRequest({ provider: 'sonilo', sourceAssetPath: '/media/uploads/cut.mp4', sourceAssetKind: 'video' }); +assert.equal(sonilo.provider, 'sonilo'); +assert.equal(sonilo.sourceAssetPath, '/media/uploads/cut.mp4'); +assert.equal(sonilo.prompt, ''); +assert.throws(() => validateSoundRequest({ provider: 'sonilo' }), /video sourceAssetId/); +assert.throws( + () => validateSoundRequest({ provider: 'sonilo', sourceAssetPath: '/media/uploads/a.mp3', sourceAssetKind: 'audio' }), + /video sourceAssetId/, +); +assert.throws( + () => validateSoundRequest({ provider: 'sonilo', sourceAssetPath: '/media/uploads/cut.mp4', sourceAssetKind: 'video', prompt: 'whoosh' }), + /prompt is not supported/, +); +assert.throws( + () => validateSoundRequest({ provider: 'sonilo', sourceAssetPath: '/media/uploads/cut.mp4', sourceAssetKind: 'video', loop: true }), + /ElevenLabs sound controls/, +); +assert.throws( + () => validateSoundRequest({ prompt: 'x', sourceAssetPath: '/media/uploads/cut.mp4', sourceAssetKind: 'video' }), + /sonilo provider only/, +); +assert.throws(() => validateSoundRequest({ provider: 'freesound', prompt: 'x' }), /elevenlabs or sonilo/); + +console.log('sound.check: ok (elevenlabs official sound parameters + sonilo video-to-sfx)'); diff --git a/src/agent/capabilities.ts b/src/agent/capabilities.ts index 96ec2761..ff3817a2 100644 --- a/src/agent/capabilities.ts +++ b/src/agent/capabilities.ts @@ -85,6 +85,11 @@ const CAP_PROVIDERS: Partial> = { music: [ { label: 'Mureka', arg: 'mureka', argKey: 'provider', need: [['MUREKA_API_KEY']] }, { label: 'MiniMax', arg: 'minimax', argKey: 'provider', need: [['MINIMAX_API_KEY']] }, + { label: 'Sonilo', arg: 'sonilo', argKey: 'provider', need: [['SONILO_API_KEY']] }, + ], + sound: [ + { label: 'ElevenLabs', arg: 'elevenlabs', argKey: 'provider', need: [['ELEVENLABS_API_KEY']] }, + { label: 'Sonilo', arg: 'sonilo', argKey: 'provider', need: [['SONILO_API_KEY']] }, ], stock: [ { label: 'Pexels', arg: 'pexels', argKey: 'provider', need: [['PEXELS_API_KEY']] }, diff --git a/src/agent/generate-workflow.ts b/src/agent/generate-workflow.ts index f7f8d955..d4a83296 100644 --- a/src/agent/generate-workflow.ts +++ b/src/agent/generate-workflow.ts @@ -18,11 +18,13 @@ export const GENERATE_WORKFLOW = ` ## Sound-effect generation - Use submit_sound only after the user explicitly requests a new/original/custom sound, or when the existing sound-effects library has no suitable result. - For ordinary whoosh, riser, impact, notification, click, ding, censor beep, record scratch, shutter, typing, or reaction sounds, use the existing library first. -- Default to 4 seconds and promptInfluence 0.3. submit_sound creates one media-pool audio asset only and does not place it on the timeline. +- Default provider elevenlabs (text prompt): default to 4 seconds and promptInfluence 0.3. Provider sonilo generates royalty-free SFX matched to a project video asset (the rendered cut, up to 3 minutes) via sourceAssetId — no prompt or duration controls. +- submit_sound creates one media-pool audio asset only and does not place it on the timeline. ## Music generation -- Use submit_music only after the user explicitly requests newly generated music; it starts an asynchronous generation job (Mureka or MiniMax). +- Use submit_music only after the user explicitly requests newly generated music; it starts an asynchronous generation job (Mureka, MiniMax, or Sonilo). - Default provider mureka and mode instrumental. Mureka also supports song (lyrics), prompt-song, soundtrack (image/video sourceAssetId), and track/stem generation (songId or audio sourceAssetId), count 1–3, styles, voice/reference IDs, ranges, and streaming tasks. MiniMax t2m supports lyrics, lyricsOptimizer, isInstrumental, sampleRate/bitrate/audioFormat; cover supports referenceAssetId or coverFeatureId plus style prompt (10–300) with a music-cover model. +- Sonilo mode v2m is video-conditioned: it reads a project video asset (the rendered cut, up to 6 minutes) via sourceAssetId and composes music matched to its pacing; prompt is a single optional style hint (≤500, works without one); exactly one result. Generated music is licensed, safe for commercial use (terms apply), and each track carries a licenseId (also archived as a .license.json sidecar beside the audio file). - Describe the style, mood, instrumentation, and intended edit context in prompt. Do not silently request extra variants. - submit_music returns immediately with a jobId. Call track_progress target=generation with action=status or action=wait; only a successful tracked result creates the media-pool audio asset. diff --git a/src/agent/skills/music/SKILL.md b/src/agent/skills/music/SKILL.md index 1d8fec2b..8d5a443c 100644 --- a/src/agent/skills/music/SKILL.md +++ b/src/agent/skills/music/SKILL.md @@ -1,7 +1,7 @@ --- name: music description: | - Music generation via Mureka and MiniMax. Use for instrumentals, songs, soundtracks, track/stem generation, or covers through `submit_music`. + Music generation via Mureka, MiniMax, and Sonilo. Use for instrumentals, songs, soundtracks, track/stem generation, covers, or video-conditioned scoring of the finished cut through `submit_music`. user-invocable: true --- @@ -15,8 +15,9 @@ user-invocable: true | --- | --- | --- | | `mureka` | `instrumental`, `song`, `prompt-song`, `soundtrack`, `track` | [references/mureka.md](references/mureka.md) | | `minimax` | `t2m`, `cover` | [references/minimax.md](references/minimax.md) | +| `sonilo` | `v2m` | [references/sonilo.md](references/sonilo.md) | -Select the named/configured vendor. Default to Mureka instrumental for ordinary BGM. Use Mureka song/prompt-song when its vocal or reference controls are wanted; use Mureka soundtrack for an image/video-driven score and track mode for generating a stem/track from a song/audio source. Use MiniMax for its text-to-music and cover models. +Select the named/configured vendor. Default to Mureka instrumental for ordinary BGM. Use Mureka song/prompt-song when its vocal or reference controls are wanted; use Mureka soundtrack for an image/video-driven score and track mode for generating a stem/track from a song/audio source. Use MiniMax for its text-to-music and cover models. Use Sonilo v2m when the music should be composed from the finished cut itself (a project video asset) rather than a text description. ## Shared workflow @@ -53,6 +54,10 @@ submit_music({ provider: "minimax", mode: "t2m", prompt: "Rainy night pop", lyri // MiniMax cover from a project audio asset submit_music({ provider: "minimax", mode: "cover", prompt: "Warm acoustic coffee-shop cover", referenceAssetId: "audioAssetId" }); + +// Sonilo video-to-music from the rendered cut (prompt optional) +submit_music({ provider: "sonilo", mode: "v2m", sourceAssetId: "renderedCutAssetId" }); +submit_music({ provider: "sonilo", mode: "v2m", sourceAssetId: "renderedCutAssetId", prompt: "Warm indie folk, no drums" }); ``` ## Rules diff --git a/src/agent/skills/music/references/sonilo.md b/src/agent/skills/music/references/sonilo.md new file mode 100644 index 00000000..032fc33f --- /dev/null +++ b/src/agent/skills/music/references/sonilo.md @@ -0,0 +1,23 @@ +# Sonilo (`provider: "sonilo"`) + +OpenChatCut wires the Sonilo video-to-music API (`/v1/video-to-music` → poll `/v1/tasks/{task_id}`). There is no model field: `/v1` always routes to the latest model server-side. + +## Mode + +| Mode | Input | Key controls | +| --- | --- | --- | +| `v2m` | project **video** `sourceAssetId` (the rendered cut, ≤6 minutes) | optional `prompt` ≤500 (single style hint; works promptless) | + +The video is uploaded server-side and the music is composed from the cut itself — pacing, scene changes, and emotional arc come from the footage, not from a text description. Exactly one `.m4a` result per job. + +## Rights + +Generated music is licensed and safe for commercial use (terms apply). Every track returns a `license_id`, surfaced as `licenseId` on the generation result and archived as a `.license.json` sidecar beside the audio file in the media uploads directory — keep it for commercial-use review. + +## Rules + +- `sourceAssetId` must resolve to a project **video** upload; images and audio are rejected. +- `count` is fixed at 1; `lyrics`, styles, stems, segment ranges, and audio-format controls belong to other providers and are rejected. +- Prompt is a style hint only ("warm indie folk, no drums"), not a scene description — the model already sees the video. +- Videos longer than 6 minutes fail fast locally before upload; trim or export a shorter cut first. +- The same `SONILO_API_KEY` also enables `submit_sound` with `provider: "sonilo"`: royalty-free sound effects generated from a project video asset (≤3 minutes), no prompt. diff --git a/src/agent/tools/generate-schemas.ts b/src/agent/tools/generate-schemas.ts index 26bbb7b1..02dc9368 100644 --- a/src/agent/tools/generate-schemas.ts +++ b/src/agent/tools/generate-schemas.ts @@ -92,29 +92,30 @@ export const GENERATE_TOOL_SCHEMAS: AgentToolSchema[] = [ }, { name: 'submit_sound', - description: 'Generate one original/custom sound effect with ElevenLabs and create an audio asset in the media pool. Does not place timeline items. For ordinary whooshes, clicks, impacts, dings, and similar editing sounds, use the existing library first.', + description: 'Generate one sound-effect audio asset in the media pool: ElevenLabs synthesizes from a text prompt; Sonilo generates SFX from a project video asset (the rendered cut, up to 3 minutes) with no prompt. Does not place timeline items. For ordinary whooshes, clicks, impacts, dings, and similar editing sounds, use the existing library first.', input_schema: { type: 'object', properties: { - prompt: { type: 'string', minLength: 1, description: 'Detailed sound description.' }, - durationSeconds: { type: 'number', minimum: 0.5, maximum: 30, description: 'Optional 0.5–30 seconds. Omit to let ElevenLabs choose the duration.' }, - promptInfluence: { type: 'number', minimum: 0, maximum: 1, description: 'Prompt adherence; defaults to 0.3.' }, - loop: { type: 'boolean', description: 'Create a seamless loop. Requires the configured eleven_text_to_sound_v2 model.' }, - outputFormat: { type: 'string', description: 'Official ElevenLabs codec_sample-rate_bitrate enum, e.g. mp3_44100_128, opus_48000_128, pcm_44100.' }, + provider: { type: 'string', enum: ['elevenlabs', 'sonilo'], description: 'Defaults to elevenlabs (text prompt). sonilo generates video-matched SFX from sourceAssetId instead of a prompt.' }, + prompt: { type: 'string', description: 'Detailed sound description. Required for elevenlabs; not used by sonilo.' }, + durationSeconds: { type: 'number', minimum: 0.5, maximum: 30, description: 'ElevenLabs only. Optional 0.5–30 seconds; omit to let ElevenLabs choose the duration.' }, + promptInfluence: { type: 'number', minimum: 0, maximum: 1, description: 'ElevenLabs only. Prompt adherence; defaults to 0.3.' }, + loop: { type: 'boolean', description: 'ElevenLabs only. Create a seamless loop; requires the configured eleven_text_to_sound_v2 model.' }, + outputFormat: { type: 'string', description: 'ElevenLabs only. Official codec_sample-rate_bitrate enum, e.g. mp3_44100_128, opus_48000_128, pcm_44100.' }, + sourceAssetId: { type: 'string', description: 'Sonilo only. Project video asset (the rendered cut, ≤3 minutes) the SFX are generated from.' }, name: { type: 'string', description: 'Optional media-pool asset name.' }, }, - required: ['prompt'], }, }, { name: 'submit_music', - description: 'Submit a Mureka or MiniMax music job. Mureka supports instrumental, lyrics song, prompt song, soundtrack, and track/stem modes with up to 3 results. MiniMax supports text-to-music and cover. Results are saved to the media pool.', + description: 'Submit a Mureka, MiniMax, or Sonilo music job. Mureka supports instrumental, lyrics song, prompt song, soundtrack, and track/stem modes with up to 3 results. MiniMax supports text-to-music and cover. Sonilo v2m generates music from a project video asset (the rendered cut, up to 6 minutes) — video-conditioned, with a single optional style prompt. Results are saved to the media pool.', input_schema: { type: 'object', properties: { - prompt: { type: 'string', maxLength: 2000, description: 'Style/control prompt. Required for most modes; optional with Mureka provider IDs. Mureka max is 1024 or 2000 by mode; MiniMax cover requires 10–300.' }, - provider: { type: 'string', enum: ['mureka', 'minimax'], description: 'Defaults to mureka.' }, - mode: { type: 'string', enum: ['instrumental', 'song', 'prompt-song', 'soundtrack', 'track', 't2m', 'cover'], description: 'Mureka: instrumental/song/prompt-song/soundtrack/track. MiniMax: t2m/cover.' }, + prompt: { type: 'string', maxLength: 2000, description: 'Style/control prompt. Required for most modes; optional with Mureka provider IDs and Sonilo v2m (≤500, works promptless). Mureka max is 1024 or 2000 by mode; MiniMax cover requires 10–300.' }, + provider: { type: 'string', enum: ['mureka', 'minimax', 'sonilo'], description: 'Defaults to mureka.' }, + mode: { type: 'string', enum: ['instrumental', 'song', 'prompt-song', 'soundtrack', 'track', 't2m', 'cover', 'v2m'], description: 'Mureka: instrumental/song/prompt-song/soundtrack/track. MiniMax: t2m/cover. Sonilo: v2m (video-to-music).' }, lyrics: { type: 'string', maxLength: 5000, description: 'Mureka song/track or MiniMax t2m/cover lyrics. Provider-specific limits are validated.' }, isInstrumental: { type: 'boolean', description: 'MiniMax t2m only. Force instrumental.' }, lyricsOptimizer: { type: 'boolean', description: 'MiniMax t2m only. Auto-generate lyrics from prompt.' }, @@ -131,7 +132,7 @@ export const GENERATE_TOOL_SCHEMAS: AgentToolSchema[] = [ instrumentalId: { type: 'string', description: 'Mureka instrumental file ID; use instead of prompt.' }, vocalId: { type: 'string', description: 'Mureka cloned vocal ID.' }, melodyId: { type: 'string', description: 'Mureka lyrics-song melody ID; cannot combine with other controls.' }, - sourceAssetId: { type: 'string', description: 'Mureka soundtrack image/video asset or track-mode audio asset.' }, + sourceAssetId: { type: 'string', description: 'Mureka soundtrack image/video asset, track-mode audio asset, or Sonilo v2m video asset (the rendered cut, ≤6 minutes).' }, audioStartMs: { type: 'integer', minimum: 0, description: 'Mureka soundtrack segment start in ms.' }, audioEndMs: { type: 'integer', minimum: 0, description: 'Mureka soundtrack segment end in ms; range must be at least 3000ms.' }, songId: { type: 'string', description: 'Mureka track source song ID; excludes sourceAssetId.' }, diff --git a/src/agent/tools/generate-tool-input.ts b/src/agent/tools/generate-tool-input.ts index cb001a56..6c813033 100644 --- a/src/agent/tools/generate-tool-input.ts +++ b/src/agent/tools/generate-tool-input.ts @@ -167,9 +167,18 @@ function murekaMusic(args: GenerateArgs): SubmitMusicArgs { }; } -const MUSIC_STRATEGIES = { mureka: murekaMusic, minimax: minimaxMusic } as const; +// Sonilo v2m is video-conditioned: the project video asset (the rendered cut) +// is the input; prompt is a single optional style hint. Everything else is +// provider-specific elsewhere and dropped here. +const soniloMusic = (args: GenerateArgs): SubmitMusicArgs => ({ + provider: 'sonilo', mode: 'v2m', prompt: str(args.prompt), name: str(args.name), + sourceAssetId: str(args.sourceAssetId), +}); + +const MUSIC_STRATEGIES = { mureka: murekaMusic, minimax: minimaxMusic, sonilo: soniloMusic } as const; export function buildSubmitMusicArgs(args: GenerateArgs): SubmitMusicArgs { - return MUSIC_STRATEGIES[args.provider === 'minimax' ? 'minimax' : 'mureka'](args); + const provider = args.provider === 'minimax' || args.provider === 'sonilo' ? args.provider : 'mureka'; + return MUSIC_STRATEGIES[provider](args); } const videoBase = (args: GenerateArgs, model: SubmitVideoArgs['model']): SubmitVideoArgs => ({ @@ -204,7 +213,15 @@ export function buildSubmitVideoArgs(args: GenerateArgs): SubmitVideoArgs { return VIDEO_STRATEGIES[model](args); } -export const buildSubmitSoundArgs = (args: GenerateArgs): SubmitSoundArgs => ({ - prompt: String(args.prompt ?? ''), durationSeconds: num(args.durationSeconds), promptInfluence: num(args.promptInfluence), - loop: bool(args.loop), outputFormat: str(args.outputFormat), name: str(args.name), -}); +export const buildSubmitSoundArgs = (args: GenerateArgs): SubmitSoundArgs => { + // Sonilo SFX come from the video itself; ElevenLabs synthesis controls + // (and the prompt) do not apply and are dropped. + if (args.provider === 'sonilo') { + return { provider: 'sonilo', sourceAssetId: str(args.sourceAssetId), name: str(args.name) }; + } + return { + provider: 'elevenlabs', + prompt: String(args.prompt ?? ''), durationSeconds: num(args.durationSeconds), promptInfluence: num(args.promptInfluence), + loop: bool(args.loop), outputFormat: str(args.outputFormat), name: str(args.name), + }; +}; diff --git a/src/agent/tools/generation-tool-input.verify.ts b/src/agent/tools/generation-tool-input.verify.ts index 2613e504..ab70bff5 100644 --- a/src/agent/tools/generation-tool-input.verify.ts +++ b/src/agent/tools/generation-tool-input.verify.ts @@ -1,5 +1,5 @@ import assert from 'node:assert/strict'; -import { buildSubmitImageArgs, buildSubmitMusicArgs, buildSubmitVideoArgs, buildSubmitVoiceArgs, shouldAddImageToTimeline } from './generate-tool-input'; +import { buildSubmitImageArgs, buildSubmitMusicArgs, buildSubmitSoundArgs, buildSubmitVideoArgs, buildSubmitVoiceArgs, shouldAddImageToTimeline } from './generate-tool-input'; assert.equal(shouldAddImageToTimeline({}), true); assert.equal(shouldAddImageToTimeline({ addToTimeline: false }), false); @@ -117,6 +117,40 @@ assert.equal(soundtrack.styles, undefined, 'soundtrack must not receive prompt-s assert.equal(soundtrack.vocalId, undefined, 'soundtrack must not receive song controls'); assert.equal(soundtrack.lyricsOptimizer, undefined, 'Mureka must not receive MiniMax controls'); +const soniloMusic = buildSubmitMusicArgs({ + provider: 'sonilo', mode: 'soundtrack', prompt: 'warm indie folk', sourceAssetId: 'cut-1', + lyrics: 'hello', styles: ['pop'], count: 2, stream: true, bitrate: 128000, audioFormat: 'mp3', +}); +assert.equal(soniloMusic.provider, 'sonilo'); +assert.equal(soniloMusic.mode, 'v2m', 'sonilo always submits v2m'); +assert.equal(soniloMusic.sourceAssetId, 'cut-1'); +assert.equal(soniloMusic.prompt, 'warm indie folk'); +assert.equal(soniloMusic.lyrics, undefined, 'sonilo must not receive lyrics'); +assert.equal(soniloMusic.styles, undefined, 'sonilo must not receive Mureka styles'); +assert.equal(soniloMusic.count, undefined, 'sonilo must not receive Mureka count'); +assert.equal(soniloMusic.stream, undefined, 'sonilo must not receive Mureka streaming'); +assert.equal(soniloMusic.bitrate, undefined, 'sonilo must not receive MiniMax audio settings'); +assert.equal(soniloMusic.audioFormat, undefined, 'sonilo output format is fixed'); + +const elevenSound = buildSubmitSoundArgs({ + prompt: 'thunder', durationSeconds: 4, promptInfluence: 0.5, loop: true, sourceAssetId: 'cut-1', +}); +assert.equal(elevenSound.provider, 'elevenlabs', 'sound provider defaults to elevenlabs'); +assert.equal(elevenSound.prompt, 'thunder'); +assert.equal(elevenSound.sourceAssetId, undefined, 'ElevenLabs must not receive a sonilo video source'); + +const soniloSound = buildSubmitSoundArgs({ + provider: 'sonilo', sourceAssetId: 'cut-1', prompt: 'whoosh', durationSeconds: 4, + promptInfluence: 0.5, loop: true, outputFormat: 'mp3_44100_128', name: 'cut sfx', +}); +assert.equal(soniloSound.provider, 'sonilo'); +assert.equal(soniloSound.sourceAssetId, 'cut-1'); +assert.equal(soniloSound.name, 'cut sfx'); +assert.equal(soniloSound.prompt, undefined, 'sonilo SFX are generated from the video, not a prompt'); +assert.equal(soniloSound.durationSeconds, undefined, 'sonilo must not receive ElevenLabs duration'); +assert.equal(soniloSound.loop, undefined, 'sonilo must not receive ElevenLabs loop'); +assert.equal(soniloSound.outputFormat, undefined, 'sonilo must not receive ElevenLabs output format'); + const eleven = buildSubmitVoiceArgs({ provider: 'elevenlabs', text: 'Hello', voiceId: 'peter', outputFormat: 'mp3_44100_128', volume: 1, sampleRate: 32000, audioFormat: 'mp3', speedRatio: 1, diff --git a/src/components/settings/settingsMediaProviders.ts b/src/components/settings/settingsMediaProviders.ts index 3a3f6754..250c7a39 100644 --- a/src/components/settings/settingsMediaProviders.ts +++ b/src/components/settings/settingsMediaProviders.ts @@ -243,4 +243,5 @@ export const ROUTE_NEEDS: Record = { kling: [['KLING_API_KEY']], hailuo: [['MINIMAX_API_KEY']], mureka: [['MUREKA_API_KEY']], + sonilo: [['SONILO_API_KEY']], }; diff --git a/src/components/settings/settingsSchema.ts b/src/components/settings/settingsSchema.ts index 0aedee44..2114f157 100644 --- a/src/components/settings/settingsSchema.ts +++ b/src/components/settings/settingsSchema.ts @@ -230,10 +230,11 @@ export const SETTINGS_CATEGORIES: readonly SettingsCategory[] = [ byteplusPage('video', modelText('BYTEPLUS_VIDEO_MODEL', '视频模型', 'seedance-1-5-pro-251215', '测试连接后可直接选择接口返回的模型，也可以手动填写模型 ID。', true), 'BytePlus · Seedance'), ] }, - { key: 'music', title: '生音乐', hint: 'submit_music · 文字生成配乐，任一厂商即可。', + { key: 'music', title: '生音乐', hint: 'submit_music · 文字 / 成片生成配乐，任一厂商即可。', route: routeSelect('PREFERRED_MUSIC_VENDOR', [ { value: 'mureka', label: 'Mureka' }, { value: 'minimax', label: 'MiniMax' }, + { value: 'sonilo', label: 'Sonilo' }, ]), vendors: [ { key: 'music/mureka', vendor: 'mureka', title: 'Mureka', fields: [ @@ -243,6 +244,14 @@ export const SETTINGS_CATEGORIES: readonly SettingsCategory[] = [ ] }, minimaxPage('music', modelSelect('MINIMAX_MUSIC_MODEL', '音乐模型', 'music-2.6', ['music-3.0', 'music-2.6', 'music-3.0-free', 'music-2.6-free', 'music-cover', 'music-cover-free'])), + { key: 'music/sonilo', vendor: 'sonilo', title: 'Sonilo', + note: '按成片生成：把渲染好的视频交给 Sonilo，配乐跟着画面节奏走（可选一句风格提示，不填也行）。' + + '配乐自带授权、可商用（以条款为准）；每条音轨附 license_id 留档。' + + '同一个 Key 也用于按成片生成音效（submit_sound，免版税）。', + fields: [ + secret('SONILO_API_KEY', 'API Key'), + text('SONILO_BASE_URL', 'Base URL', '默认 https://api.sonilo.com'), + ] }, ] }, ], }, diff --git a/src/components/settings/vendorIcons.tsx b/src/components/settings/vendorIcons.tsx index 1561c6d1..7bfcbb17 100644 --- a/src/components/settings/vendorIcons.tsx +++ b/src/components/settings/vendorIcons.tsx @@ -40,7 +40,7 @@ import visionSvg from '../../../assets/vendor-icons/vision.svg?raw'; export type VendorId = | 'llm' | 'anthropic' | 'openai' | 'gemini' | 'kimi' | 'qwen' | 'glm' | 'deepseek' | 'mistral' | 'openrouter' | 'ollama' | 'lmstudio' | 'xiaomi' | 'minimax' | 'hailuo' | 'elevenlabs' | 'doubao' - | 'seedance' | 'kling' | 'mureka' | 'pexels' | 'pixabay' | 'unsplash' | 'freesound' + | 'seedance' | 'kling' | 'mureka' | 'sonilo' | 'pexels' | 'pixabay' | 'unsplash' | 'freesound' | 'assemblyai' | 'deepgram' | 'groq' | 'cartesia' | 'e2b' | 'firecrawl' | 'r2' | 'localdisk' | 'localasr' | 'stepfun' | 'byteplus' | 'inworld' | 'fishaudio' | 'speechify' | 'wavespeed' | 'vision' | 'proxy'; @@ -99,6 +99,7 @@ const MONOGRAMS: Partial crypto.randomUUID?.() ?? `generated_${Date.now()}_${Math.random().toString(36).slice(2)}`; +function resolveVideoAsset(ref: string, state: TimelineState): MediaAsset { + const clean = ref.replace(/^asset:\/\//, '').trim(); + const asset = (state.assets ?? []).find( + (a) => a.id === clean || a.id.startsWith(clean) || a.name === clean || a.src === clean, + ); + if (!asset) throw new Error(`sound source asset not found: ${ref}`); + if (asset.kind !== 'video') throw new Error(`sound source asset is not video: ${ref}`); + let pathname = asset.src; + if (pathname.startsWith('http')) { + const url = new URL(pathname, location.origin); + if (url.origin !== location.origin) throw new Error(`external video URLs are not accepted: ${ref}`); + pathname = url.pathname; + } + if (!pathname.startsWith('/media/uploads/')) throw new Error(`sound source must be a project upload: ${ref}`); + return { ...asset, src: pathname }; +} + export async function submitSound(args: SubmitSoundArgs, state: TimelineState): Promise { - const prompt = args.prompt.trim(); - if (!prompt) throw new Error('prompt is required'); + const provider = args.provider === 'sonilo' ? 'sonilo' : 'elevenlabs'; + const prompt = args.prompt?.trim() ?? ''; + if (provider === 'elevenlabs' && !prompt) throw new Error('prompt is required'); + const sourceAsset = provider === 'sonilo' && args.sourceAssetId + ? resolveVideoAsset(args.sourceAssetId, state) + : undefined; const response = await fetch('/generate/sound', { method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ ...args, prompt }), + body: JSON.stringify({ + ...args, + provider, + prompt, + sourceAssetPath: sourceAsset?.src, + sourceAssetKind: sourceAsset?.kind, + sourceAssetId: undefined, + }), }); const result = await response.json().catch(() => ({})) as SoundResponse; if (!response.ok) throw new Error(result.error ?? `sound generation failed (${response.status})`); if (!result.path || !result.durationSeconds) throw new Error('sound generation returned invalid audio'); + const fallbackName = provider === 'sonilo' + ? `SFX · ${(sourceAsset?.name ?? 'cut').slice(0, 36)}` + : `Sound · ${prompt.slice(0, 36)}`; return { id: newId(), - name: args.name?.trim() || `Sound · ${prompt.slice(0, 36)}`, + name: args.name?.trim() || fallbackName, kind: 'audio', src: result.path, durationInFrames: Math.max(1, Math.round(result.durationSeconds * state.fps)), diff --git a/src/i18n/dict/en/settings.ts b/src/i18n/dict/en/settings.ts index c242614a..63fbe123 100644 --- a/src/i18n/dict/en/settings.ts +++ b/src/i18n/dict/en/settings.ts @@ -51,7 +51,7 @@ export default { 'submit_image · 文生图 / 图生图，任一厂商即可。': 'submit_image · Text-to-image / image-to-image; any one vendor works.', 'submit_voice · 文字转配音，任一厂商即可。': 'submit_voice · Text to voiceover; any one vendor works.', 'submit_video · 文 / 图生视频，任一厂商即可。': 'submit_video · Text / image to video; any one vendor works.', - 'submit_music · 文字生成配乐，任一厂商即可。': 'submit_music · Text to soundtrack; any one vendor works.', + 'submit_music · 文字 / 成片生成配乐，任一厂商即可。': 'submit_music · Text or finished cut to soundtrack; any one vendor works.', 'search_stock_media · 搜索可商用图片 / 视频素材。': 'search_stock_media · Search commercially usable photos / videos.', 'transcribe_track · 词级字幕、清口水、删词。': 'transcribe_track · Word-level captions, filler cleanup, word deletion.', '素材的本地保存目录，与可选的 R2 云备份。': 'Local save directory for media, plus optional R2 cloud backup.', @@ -129,6 +129,8 @@ export default { // ──Page Note/Field Note── 'MiniMax 同一个 Key，配置一次全能力（生图 / 配音 / 视频 / 音乐）通用。': 'One MiniMax key covers every capability (image / voice / video / music) — configure once.', 'Key 同时用于音效生成（submit_sound）。': 'The key is also used for sound-effect generation (submit_sound).', + '按成片生成：把渲染好的视频交给 Sonilo，配乐跟着画面节奏走（可选一句风格提示，不填也行）。配乐自带授权、可商用（以条款为准）；每条音轨附 license_id 留档。同一个 Key 也用于按成片生成音效（submit_sound，免版税）。': + 'Generates from the finished cut: hand Sonilo the rendered video and the music follows its pacing (one optional style hint; works without one). Music is licensed, safe for commercial use (terms apply); each track comes with a license_id record. The same key also powers video-matched sound effects (submit_sound, royalty-free).', '可直接使用 Anthropic 官方 API Key；如使用兼容服务，再修改 Base URL 和模型。': 'Use an official Anthropic API key directly, or change the Base URL and model for a compatible service.', '默认使用 Claude Fable 5；自定义兼容地址时，也可填写该服务支持的模型 ID。': 'Claude Fable 5 is the default. With a compatible endpoint, enter any model ID supported by that service.', '每个厂商独立保存地址、密钥与模型。先测试连接，成功后可从接口返回的模型中选择。': 'Each provider keeps its own endpoint, key, and model. Test the connection, then choose from the models returned by that API.', @@ -262,6 +264,7 @@ export default { '默认 https://ark.cn-beijing.volces.com/api/v3': 'Default https://ark.cn-beijing.volces.com/api/v3', '默认 https://api-singapore.klingai.com': 'Default https://api-singapore.klingai.com', '默认 https://api.mureka.ai': 'Default https://api.mureka.ai', + '默认 https://api.sonilo.com': 'Default https://api.sonilo.com', '默认 https://api.minimaxi.com': 'Default https://api.minimaxi.com', '默认 https://api.mistral.ai/v1': 'Default https://api.mistral.ai/v1', '默认 https://api.groq.com/openai/v1': 'Default https://api.groq.com/openai/v1',