diff --git a/plugins/runway-api/LICENSE.runway-api b/plugins/runway-api/LICENSE.runway-api new file mode 100644 index 00000000..7aa33638 --- /dev/null +++ b/plugins/runway-api/LICENSE.runway-api @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 RunwayML, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/plugins/runway-api/README.md b/plugins/runway-api/README.md new file mode 100644 index 00000000..ef9f6c69 --- /dev/null +++ b/plugins/runway-api/README.md @@ -0,0 +1,36 @@ +# runway-api + +Runway API skills for Cline users building or operating media generation workflows. + +## What It Does + +This plugin bundles skills for two related workflows: + +- Generating media directly with Runway's API, including text/image/video generation, audio generation, uploads, organization details, and general API calls. +- Integrating Runway features into server-side apps, including compatibility checks, API key setup, video/image/audio endpoints, uploads, real-time avatar characters, document knowledge, and React avatar embeds. + +The plugin also includes the Python and Node helper scripts used by the direct-generation skills, plus bundled guidance for paid API calls, media uploads, outbound URL fetches, and API-key handling. + +## Install + +```bash +cline plugin install runway-api +``` + +For local development from this repository: + +```bash +cline plugin install ./plugins/runway-api --cwd . +``` + +## Requirements + +- A Runway developer account with available credits. +- `RUNWAYML_API_SECRET` in the environment or a user-created local `.env` file when making API calls. +- `uv` for the bundled Python generation helper scripts. +- Node.js 20 or newer for the general `use-runway-api` helper script. +- A server-side app when using integration skills. API keys must not be exposed in frontend code. + +## Security Notes + +Runway operations can spend credits and may upload or generate sensitive media. Cline should confirm paid generation, uploads, organization/account actions, external media URLs, output locations, and production code changes before acting. diff --git a/plugins/runway-api/assets/logo.png b/plugins/runway-api/assets/logo.png new file mode 100644 index 00000000..9a4f1f6d Binary files /dev/null and b/plugins/runway-api/assets/logo.png differ diff --git a/plugins/runway-api/index.ts b/plugins/runway-api/index.ts new file mode 100644 index 00000000..85082753 --- /dev/null +++ b/plugins/runway-api/index.ts @@ -0,0 +1,10 @@ +import type { AgentPlugin } from "@cline/sdk" + +const plugin: AgentPlugin = { + name: "runway-api", + manifest: { + capabilities: ["skills"], + }, +} + +export default plugin diff --git a/plugins/runway-api/package.json b/plugins/runway-api/package.json new file mode 100644 index 00000000..2fcfa05f --- /dev/null +++ b/plugins/runway-api/package.json @@ -0,0 +1,19 @@ +{ + "name": "runway-api", + "version": "0.0.0", + "private": true, + "type": "module", + "description": "Cline plugin that bundles Runway API media generation and app integration skills.", + "cline": { + "plugins": [ + { + "paths": [ + "./index.ts" + ], + "capabilities": [ + "skills" + ] + } + ] + } +} diff --git a/plugins/runway-api/scripts/generate_audio.py b/plugins/runway-api/scripts/generate_audio.py new file mode 100644 index 00000000..73e76542 --- /dev/null +++ b/plugins/runway-api/scripts/generate_audio.py @@ -0,0 +1,130 @@ +# /// script +# requires-python = ">=3.10" +# dependencies = ["requests"] +# /// + +"""Generate audio using the Runway API (TTS, sound effects, voice isolation, dubbing).""" + +import argparse +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from runway_helpers import ( + get_api_key, + api_post, + poll_task, + download_file, + ensure_url, + output_path, +) + +AUDIO_TYPES = { + "tts": { + "endpoint": "/v1/text_to_speech", + "model": "eleven_multilingual_v2", + "description": "Text to speech", + }, + "sfx": { + "endpoint": "/v1/sound_effect", + "model": "eleven_text_to_sound_v2", + "description": "Sound effect generation", + }, + "isolate": { + "endpoint": "/v1/voice_isolation", + "model": "eleven_voice_isolation", + "description": "Isolate voice from audio", + }, + "dub": { + "endpoint": "/v1/voice_dubbing", + "model": "eleven_voice_dubbing", + "description": "Dub to another language", + }, + "sts": { + "endpoint": "/v1/speech_to_speech", + "model": "eleven_multilingual_sts_v2", + "description": "Voice conversion", + }, +} + + +def main(): + parser = argparse.ArgumentParser(description="Generate audio with the Runway API") + parser.add_argument("--filename", required=True, help="Output filename (e.g. output.mp3)") + parser.add_argument( + "--type", + required=True, + choices=list(AUDIO_TYPES.keys()), + help="Audio type: tts, sfx, isolate, dub, sts", + ) + parser.add_argument("--text", help="Text input (required for tts and sfx)") + parser.add_argument("--audio-url", help="Audio URL or local path (for isolate, dub, sts)") + parser.add_argument("--voice-id", help="Voice ID (for tts and sts)") + parser.add_argument("--target-language", help="Target language code (for dub, e.g. 'es')") + parser.add_argument("--output-dir", help="Output directory (default: cwd)") + args = parser.parse_args() + + api_key = get_api_key() + audio_type = AUDIO_TYPES[args.type] + endpoint = audio_type["endpoint"] + model = audio_type["model"] + + body = {"model": model} + + if args.type == "tts": + if not args.text: + print("Error: --text is required for tts.", file=sys.stderr) + sys.exit(1) + body["promptText"] = args.text + body["voice"] = {"type": "runway-preset", "presetId": args.voice_id or "Maya"} + + elif args.type == "sfx": + if not args.text: + print("Error: --text is required for sfx.", file=sys.stderr) + sys.exit(1) + body["promptText"] = args.text + + elif args.type == "isolate": + if not args.audio_url: + print("Error: --audio-url is required for isolate.", file=sys.stderr) + sys.exit(1) + body["audioUri"] = ensure_url(args.audio_url, api_key) + + elif args.type == "sts": + if not args.audio_url: + print("Error: --audio-url is required for sts.", file=sys.stderr) + sys.exit(1) + audio_uri = ensure_url(args.audio_url, api_key) + body["media"] = {"type": "audio", "uri": audio_uri} + body["voice"] = {"type": "runway-preset", "presetId": args.voice_id or "Maya"} + + elif args.type == "dub": + if not args.audio_url: + print("Error: --audio-url is required for dub.", file=sys.stderr) + sys.exit(1) + if not args.target_language: + print("Error: --target-language is required for dub.", file=sys.stderr) + sys.exit(1) + body["audioUri"] = ensure_url(args.audio_url, api_key) + body["targetLang"] = args.target_language + + print(f"Generating audio ({args.type}) with {model}...", file=sys.stderr) + task = api_post(api_key, endpoint, body) + task_id = task.get("id") + print(f"Task created: {task_id}", file=sys.stderr) + + result = poll_task(api_key, task_id) + urls = result.get("output", []) + + if not urls: + print("Error: No output URLs in result.", file=sys.stderr) + sys.exit(1) + + out = output_path(args.filename, args.output_dir) + path = download_file(urls[0], out) + print(path) + print(f"Saved: {path}", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/plugins/runway-api/scripts/generate_image.py b/plugins/runway-api/scripts/generate_image.py new file mode 100644 index 00000000..93d6365a --- /dev/null +++ b/plugins/runway-api/scripts/generate_image.py @@ -0,0 +1,103 @@ +# /// script +# requires-python = ">=3.10" +# dependencies = ["requests"] +# /// + +"""Generate images using the Runway API.""" + +import argparse +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from runway_helpers import ( + get_api_key, + api_post, + poll_task, + download_file, + ensure_url, + output_path, + IMAGE_MODELS, +) + + +def main(): + parser = argparse.ArgumentParser(description="Generate images with the Runway API") + parser.add_argument("--prompt", required=True, help="Text description of the image") + parser.add_argument( + "--filename", required=True, help="Output filename (e.g. output.png)" + ) + parser.add_argument( + "--model", + default="gemini_2.5_flash", + choices=list(IMAGE_MODELS.keys()), + help="Image model (default: gemini_2.5_flash / Nano Banana)", + ) + parser.add_argument( + "--ratio", default=None, help="Aspect ratio. gemini_2.5_flash: 1344:768, 768:1344, 1024:1024, etc. Others: 1280:720" + ) + parser.add_argument( + "--reference-images", + nargs="*", + metavar="TAG=URL", + help="Reference images as Tag=URL pairs (e.g. Style=https://...)", + ) + parser.add_argument("--output-dir", help="Output directory (default: cwd)") + args = parser.parse_args() + + api_key = get_api_key() + + if args.ratio: + ratio = args.ratio + elif args.model == "gemini_2.5_flash": + ratio = "1344:768" + else: + ratio = "1280:720" + + body = { + "model": args.model, + "promptText": args.prompt, + "ratio": ratio, + } + + if args.reference_images: + refs = [] + for pair in args.reference_images: + if "=" not in pair: + print( + f"Error: Reference image must be Tag=URL, got: {pair}", + file=sys.stderr, + ) + sys.exit(1) + tag, source = pair.split("=", 1) + refs.append({"tag": tag, "uri": ensure_url(source, api_key)}) + body["referenceImages"] = refs + elif args.model == "gen4_image_turbo": + print("Error: gen4_image_turbo requires --reference-images.", file=sys.stderr) + sys.exit(1) + + print(f"Generating image with {args.model}...", file=sys.stderr) + task = api_post(api_key, "/v1/text_to_image", body) + task_id = task.get("id") + print(f"Task created: {task_id}", file=sys.stderr) + + result = poll_task(api_key, task_id) + urls = result.get("output", []) + + if not urls: + print("Error: No output URLs in result.", file=sys.stderr) + sys.exit(1) + + for i, url in enumerate(urls): + if len(urls) == 1: + out = output_path(args.filename, args.output_dir) + else: + base, ext = os.path.splitext(args.filename) + out = output_path(f"{base}-{i + 1}{ext}", args.output_dir) + path = download_file(url, out) + print(path) + print(f"Saved: {path}", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/plugins/runway-api/scripts/generate_video.py b/plugins/runway-api/scripts/generate_video.py new file mode 100644 index 00000000..fbffa5a4 --- /dev/null +++ b/plugins/runway-api/scripts/generate_video.py @@ -0,0 +1,117 @@ +# /// script +# requires-python = ">=3.10" +# dependencies = ["requests"] +# /// + +"""Generate videos using the Runway API.""" + +import argparse +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from runway_helpers import ( + get_api_key, + api_post, + poll_task, + download_file, + ensure_url, + output_path, + VIDEO_MODELS, +) + + +def main(): + parser = argparse.ArgumentParser(description="Generate videos with the Runway API") + parser.add_argument("--prompt", required=True, help="Text description of the video") + parser.add_argument("--filename", required=True, help="Output filename (e.g. output.mp4)") + parser.add_argument( + "--model", + default="gen4.5", + choices=list(VIDEO_MODELS.keys()), + help="Video model (default: gen4.5)", + ) + parser.add_argument("--ratio", default="1280:720", help="Aspect ratio (default: 1280:720). All models use pixel-based ratios.") + parser.add_argument("--duration", type=int, default=5, help="Duration in seconds (default: 5)") + parser.add_argument("--image-url", help="Input image URL or local path for image-to-video") + parser.add_argument("--video-url", help="Input video URL or local path for video-to-video (gen4_aleph, seedance2)") + parser.add_argument("--output-dir", help="Output directory (default: cwd)") + args = parser.parse_args() + + api_key = get_api_key() + model_info = VIDEO_MODELS[args.model] + + valid_durations = model_info.get("durations") + duration = args.duration + if valid_durations and duration not in valid_durations: + closest = min(valid_durations, key=lambda d: abs(d - duration)) + print(f" Note: {args.model} supports durations {valid_durations}, using {closest}s instead of {duration}s.", file=sys.stderr) + duration = closest + + if args.video_url: + if "video_to_video" not in model_info["endpoints"]: + print(f"Error: {args.model} does not support video-to-video.", file=sys.stderr) + sys.exit(1) + endpoint = "/v1/video_to_video" + video_uri = ensure_url(args.video_url, api_key) + if args.model == "seedance2": + body = { + "model": args.model, + "promptVideo": video_uri, + "promptText": args.prompt, + } + else: + body = { + "model": args.model, + "videoUri": video_uri, + "promptText": args.prompt, + } + elif args.image_url: + if "image_to_video" not in model_info["endpoints"]: + print(f"Error: {args.model} does not support image-to-video.", file=sys.stderr) + sys.exit(1) + endpoint = "/v1/image_to_video" + image_uri = ensure_url(args.image_url, api_key) + body = { + "model": args.model, + "promptImage": image_uri, + "promptText": args.prompt, + "ratio": args.ratio, + } + body["duration"] = duration + else: + if "text_to_video" not in model_info["endpoints"]: + print( + f"Error: {args.model} requires an input image (--image-url). " + "It does not support text-only generation.", + file=sys.stderr, + ) + sys.exit(1) + endpoint = "/v1/text_to_video" + body = { + "model": args.model, + "promptText": args.prompt, + "ratio": args.ratio, + "duration": duration, + } + + print(f"Generating video with {args.model} ({args.duration}s, {args.ratio})...", file=sys.stderr) + task = api_post(api_key, endpoint, body) + task_id = task.get("id") + print(f"Task created: {task_id}", file=sys.stderr) + + result = poll_task(api_key, task_id) + urls = result.get("output", []) + + if not urls: + print("Error: No output URLs in result.", file=sys.stderr) + sys.exit(1) + + out = output_path(args.filename, args.output_dir) + path = download_file(urls[0], out) + print(path) + print(f"Saved: {path}", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/plugins/runway-api/scripts/get_task.py b/plugins/runway-api/scripts/get_task.py new file mode 100644 index 00000000..2b8635b7 --- /dev/null +++ b/plugins/runway-api/scripts/get_task.py @@ -0,0 +1,35 @@ +# /// script +# requires-python = ">=3.10" +# dependencies = ["requests"] +# /// + +"""Check the status of a Runway API task.""" + +import argparse +import json +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from runway_helpers import get_api_key, api_get + + +def main(): + parser = argparse.ArgumentParser(description="Check Runway task status") + parser.add_argument("--task-id", required=True, help="Task ID to check") + parser.add_argument("--wait", action="store_true", help="Poll until the task completes") + args = parser.parse_args() + + api_key = get_api_key() + + if args.wait: + from runway_helpers import poll_task + result = poll_task(api_key, args.task_id) + print(json.dumps(result, indent=2)) + else: + task = api_get(api_key, f"/v1/tasks/{args.task_id}") + print(json.dumps(task, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/plugins/runway-api/scripts/list_models.py b/plugins/runway-api/scripts/list_models.py new file mode 100644 index 00000000..fc6537ab --- /dev/null +++ b/plugins/runway-api/scripts/list_models.py @@ -0,0 +1,55 @@ +# /// script +# requires-python = ">=3.10" +# dependencies = ["requests"] +# /// + +"""List available Runway API models and their costs.""" + +import argparse +import json +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from runway_helpers import VIDEO_MODELS, IMAGE_MODELS, AUDIO_MODELS + + +def print_table(title, models, extra_cols=None): + print(f"\n{'=' * 60}") + print(f" {title}") + print(f"{'=' * 60}") + for name, info in models.items(): + desc = info.get("description", "") + cost = info.get("cost", "") + print(f" {name:<30} {cost:<20} {desc}") + print() + + +def main(): + parser = argparse.ArgumentParser(description="List available Runway API models") + parser.add_argument("--type", choices=["video", "image", "audio", "all"], default="all", help="Model type to list") + parser.add_argument("--json", action="store_true", help="Output as JSON") + args = parser.parse_args() + + data = {} + if args.type in ("video", "all"): + data["video"] = VIDEO_MODELS + if args.type in ("image", "all"): + data["image"] = IMAGE_MODELS + if args.type in ("audio", "all"): + data["audio"] = AUDIO_MODELS + + if args.json: + print(json.dumps(data, indent=2)) + return + + if "video" in data: + print_table("Video Models", VIDEO_MODELS) + if "image" in data: + print_table("Image Models", IMAGE_MODELS) + if "audio" in data: + print_table("Audio Models", AUDIO_MODELS) + + +if __name__ == "__main__": + main() diff --git a/plugins/runway-api/scripts/runway_helpers.py b/plugins/runway-api/scripts/runway_helpers.py new file mode 100644 index 00000000..36646314 --- /dev/null +++ b/plugins/runway-api/scripts/runway_helpers.py @@ -0,0 +1,396 @@ +# /// script +# requires-python = ">=3.10" +# dependencies = ["requests"] +# /// + +"""Shared helpers for Runway API scripts: API calls, task polling, retry, download, error handling.""" + +import json +import os +import platform +import shutil +import subprocess +import sys +import time +import mimetypes +from urllib.parse import urlparse +import requests + +API_BASE = "https://api.dev.runwayml.com" +API_VERSION = "2024-11-06" + +# - Models registry - + +VIDEO_MODELS = { + "seedance2": { + "endpoints": ["text_to_video", "image_to_video", "video_to_video"], + "cost": "36 credits/sec", + "description": "Reference image and video, long duration (up to 15s)", + "input": "Text, Image, and/or Video", + }, + "gen4.5": { + "endpoints": ["text_to_video", "image_to_video"], + "cost": "12 credits/sec", + "description": "High quality, general purpose", + "input": "Text and/or Image", + }, + "gen4_turbo": { + "endpoints": ["image_to_video"], + "cost": "5 credits/sec", + "description": "Fast, image-driven (image required)", + "input": "Image required", + }, + "gen4_aleph": { + "endpoints": ["video_to_video"], + "cost": "15 credits/sec", + "description": "Video editing/transformation", + "input": "Video + Text/Image", + }, + "veo3": { + "endpoints": ["text_to_video", "image_to_video"], + "cost": "40 credits/sec", + "description": "Premium quality", + "input": "Text/Image", + "durations": [8], + }, + "veo3.1": { + "endpoints": ["text_to_video", "image_to_video"], + "cost": "20-40 credits/sec", + "description": "High quality Google model", + "input": "Text/Image", + "durations": [4, 6, 8], + }, + "veo3.1_fast": { + "endpoints": ["text_to_video", "image_to_video"], + "cost": "10-15 credits/sec", + "description": "Fast Google model", + "input": "Text/Image", + "durations": [4, 6, 8], + }, +} + +IMAGE_MODELS = { + "gen4_image": { + "endpoint": "text_to_image", + "cost": "5-8 credits", + "description": "Highest quality", + }, + "gen4_image_turbo": { + "endpoint": "text_to_image", + "cost": "2 credits", + "description": "Fast and cheap", + }, + "gemini_2.5_flash": { + "endpoint": "text_to_image", + "cost": "5 credits", + "description": "Google Gemini model", + }, +} + +AUDIO_MODELS = { + "eleven_multilingual_v2": { + "endpoint": "text_to_speech", + "cost": "1 credit/50 chars", + "description": "Text to speech", + }, + "eleven_text_to_sound_v2": { + "endpoint": "sound_effect", + "cost": "1-2 credits", + "description": "Sound effect generation", + }, + "eleven_voice_isolation": { + "endpoint": "voice_isolation", + "cost": "1 credit/6 sec", + "description": "Isolate voice from audio", + }, + "eleven_voice_dubbing": { + "endpoint": "voice_dubbing", + "cost": "1 credit/2 sec", + "description": "Dub to other languages", + }, + "eleven_multilingual_sts_v2": { + "endpoint": "speech_to_speech", + "cost": "1 credit/3 sec", + "description": "Voice conversion", + }, +} + +# - API key - + +def get_api_key(): + """Read the API key from RUNWAYML_API_SECRET. CLI flags are not supported to avoid exposing + the secret in shell history or process lists.""" + key = os.environ.get("RUNWAYML_API_SECRET") + if not key: + print( + "Error: RUNWAYML_API_SECRET is not set.\n" + "Export it in your shell (e.g. `export RUNWAYML_API_SECRET=...`) - do not pass keys as CLI flags.\n" + "Get your key at https://dev.runwayml.com/", + file=sys.stderr, + ) + sys.exit(1) + return key + + +def _headers(api_key): + return { + "Authorization": f"Bearer {api_key}", + "X-Runway-Version": API_VERSION, + "Content-Type": "application/json", + } + +# - Error formatting - + +def format_api_error(status_code, response_text): + msg = f"API error {status_code}" + try: + data = json.loads(response_text) + error = data.get("error", data.get("message", "")) + issues = data.get("issues", []) + except (json.JSONDecodeError, TypeError): + error = response_text[:500] if response_text else "" + issues = [] + + if status_code == 400: + detail = error + if issues: + parts = [f"{i.get('path', ['?'])[-1]}: {i.get('message', '')}" for i in issues] + detail = f"{error} [{'; '.join(parts)}]" + return f"{msg}: Invalid input - {detail}" + elif status_code == 401: + return f"{msg}: Authentication failed. Check RUNWAYML_API_SECRET." + elif status_code == 429: + return f"{msg}: Rate limited. Will retry..." + elif status_code in (502, 503, 504): + return f"{msg}: Server overload. Will retry..." + return f"{msg}: {error}" + +# - API calls with retry - + +def api_post(api_key, endpoint, body, max_retries=3): + """POST to the Runway API with automatic retry on 429/5xx.""" + headers = _headers(api_key) + delays = [5, 15, 45] + + for attempt in range(max_retries + 1): + r = requests.post(f"{API_BASE}{endpoint}", headers=headers, json=body) + if r.ok: + return r.json() + if r.status_code in (429, 502, 503, 504) and attempt < max_retries: + delay = delays[min(attempt, len(delays) - 1)] + print( + f" {format_api_error(r.status_code, r.text)}\n" + f" Retrying in {delay}s (attempt {attempt + 1}/{max_retries})...", + file=sys.stderr, + ) + time.sleep(delay) + continue + msg = format_api_error(r.status_code, r.text) + print(f"Error: {msg}", file=sys.stderr) + sys.exit(1) + + +def api_get(api_key, path, max_retries=3): + """GET from the Runway API with automatic retry on 429/5xx.""" + headers = _headers(api_key) + delays = [5, 15, 45] + + for attempt in range(max_retries + 1): + r = requests.get(f"{API_BASE}{path}", headers=headers) + if r.ok: + return r.json() + if r.status_code in (429, 502, 503, 504) and attempt < max_retries: + delay = delays[min(attempt, len(delays) - 1)] + print(f" Retrying in {delay}s...", file=sys.stderr) + time.sleep(delay) + continue + msg = format_api_error(r.status_code, r.text) + print(f"Error: {msg}", file=sys.stderr) + sys.exit(1) + +# - Task polling - + +def poll_task(api_key, task_id, interval=5, timeout=600): + """Poll a Runway task until it reaches a terminal state.""" + start = time.time() + while time.time() - start < timeout: + task = api_get(api_key, f"/v1/tasks/{task_id}") + status = task.get("status", "") + + if status == "SUCCEEDED": + return task + if status == "FAILED": + failure = task.get("failure", "Unknown error") + failure_code = task.get("failureCode", "") + detail = f"{failure_code}: {failure}" if failure_code else str(failure) + print(f"Error: Task failed - {detail}", file=sys.stderr) + sys.exit(1) + if status == "CANCELLED": + print("Error: Task was cancelled.", file=sys.stderr) + sys.exit(1) + + elapsed = int(time.time() - start) + print(f" [{task_id[:12]}] {status} ({elapsed}s)...", file=sys.stderr) + time.sleep(interval) + + print(f"Error: Task timed out after {timeout}s.", file=sys.stderr) + sys.exit(1) + +# - File download - + +def download_file(url, filename): + """Download a URL to a local file.""" + parent = os.path.dirname(filename) + if parent: + os.makedirs(parent, exist_ok=True) + r = requests.get(url, stream=True) + r.raise_for_status() + with open(filename, "wb") as f: + for chunk in r.iter_content(chunk_size=8192): + f.write(chunk) + return os.path.abspath(filename) + +# - Upload helper - + +def upload_file(api_key, local_path): + """Upload a local file to Runway and return the runway:// URI. + + Two-step process: + 1. POST /v1/uploads with filename to get a presigned uploadUrl + fields + runwayUri + 2. POST the file to uploadUrl with the returned fields as multipart form data + """ + if not os.path.isfile(local_path): + print(f"Error: File not found: {local_path}", file=sys.stderr) + sys.exit(1) + + filename = os.path.basename(local_path) + + r = requests.post( + f"{API_BASE}/v1/uploads", + headers=_headers(api_key), + json={"filename": filename, "type": "ephemeral"}, + ) + if not r.ok: + msg = format_api_error(r.status_code, r.text) + print(f"Error creating upload: {msg}", file=sys.stderr) + sys.exit(1) + + data = r.json() + upload_url = data.get("uploadUrl") + fields = data.get("fields", {}) + runway_uri = data.get("runwayUri") + + if not upload_url or not runway_uri: + print(f"Error: Upload response missing uploadUrl or runwayUri: {json.dumps(data)}", file=sys.stderr) + sys.exit(1) + + mime_type = mimetypes.guess_type(local_path)[0] or "application/octet-stream" + with open(local_path, "rb") as f: + r2 = requests.post( + upload_url, + data=fields, + files={"file": (filename, f, mime_type)}, + ) + + if not r2.ok: + print(f"Error uploading file: {r2.status_code} {r2.text[:500]}", file=sys.stderr) + sys.exit(1) + + print(f" Uploaded: {runway_uri}", file=sys.stderr) + return runway_uri + + +def _assert_safe_media_url(url): + """Validate an external media URL before sending it to the Runway API. + + Rejects non-http(s) schemes (e.g. file://, data:) and, when + RUNWAY_ALLOWED_MEDIA_HOSTS is set, enforces a comma-separated host allowlist. + Prefer uploading local files (which produce runway:// URIs) over passing + arbitrary external URLs - see the `rw-integrate-uploads` skill. + """ + parsed = urlparse(url) + if parsed.scheme not in ("http", "https"): + print( + f"Error: Unsupported URL scheme '{parsed.scheme}://'. " + "Only http(s) URLs or runway:// URIs from uploads are allowed.", + file=sys.stderr, + ) + sys.exit(1) + if not parsed.netloc: + print(f"Error: URL has no host: {url}", file=sys.stderr) + sys.exit(1) + + allowlist = os.environ.get("RUNWAY_ALLOWED_MEDIA_HOSTS", "").strip() + if allowlist: + allowed = {h.strip().lower() for h in allowlist.split(",") if h.strip()} + host = parsed.hostname.lower() if parsed.hostname else "" + if host not in allowed: + print( + f"Error: Host '{host}' is not in RUNWAY_ALLOWED_MEDIA_HOSTS.\n" + f"Allowed: {', '.join(sorted(allowed))}", + file=sys.stderr, + ) + sys.exit(1) + + if parsed.scheme == "http": + print( + f" Warning: Using insecure http:// URL ({parsed.hostname}). Prefer https or upload the file.", + file=sys.stderr, + ) + + +def ensure_url(path_or_url, api_key): + """Resolve a user-supplied input to a URI the API can fetch. + + - runway:// URIs pass through. + - http(s) URLs are validated (see `_assert_safe_media_url`) then passed through. + - Anything else is treated as a local file path and uploaded. + """ + if path_or_url.startswith("runway://"): + return path_or_url + if path_or_url.startswith("http://") or path_or_url.startswith("https://"): + _assert_safe_media_url(path_or_url) + return path_or_url + return upload_file(api_key, path_or_url) + +# - Output path helper - + +def output_path(filename, output_dir=None): + if output_dir: + os.makedirs(output_dir, exist_ok=True) + return os.path.join(output_dir, os.path.basename(filename)) + return filename + +# - Cost estimation - + +def estimate_video_credits(model, duration): + """Rough credit estimate for a video generation.""" + cost_str = VIDEO_MODELS.get(model, {}).get("cost", "") + try: + per_sec = int(cost_str.split()[0].replace("-", "").strip("~")) + except (ValueError, IndexError): + return None + return per_sec * duration + + +def estimate_image_credits(model): + cost_str = IMAGE_MODELS.get(model, {}).get("cost", "") + try: + return int(cost_str.split()[0].replace("-", "").strip("~")) + except (ValueError, IndexError): + return None + +# - Desktop notification - + +def send_notification(title, message): + try: + system = platform.system() + if system == "Linux" and shutil.which("notify-send"): + subprocess.run(["notify-send", title, message], timeout=5) + elif system == "Darwin": + script = f'display notification "{message}" with title "{title}"' + subprocess.run(["osascript", "-e", script], timeout=5) + else: + print("\a", end="", file=sys.stderr) + except Exception: + pass diff --git a/plugins/runway-api/skills/rw-api-reference/SKILL.md b/plugins/runway-api/skills/rw-api-reference/SKILL.md new file mode 100644 index 00000000..cd101f8a --- /dev/null +++ b/plugins/runway-api/skills/rw-api-reference/SKILL.md @@ -0,0 +1,423 @@ +--- +name: rw-api-reference +description: "Complete reference for Runway's public API: models, endpoints, costs, limits, and types" +user-invocable: false +--- + +# Runway Public API Reference + +> PREREQUISITE: Run `rw-check-compatibility` first to ensure the project has server-side capability. + +Base URL: `https://api.dev.runwayml.com` + +All requests require these headers: +``` +Authorization: Bearer +X-Runway-Version: 2024-11-06 +``` + +--- + +## Models & Endpoints + +### Video Generation + +| Model | Endpoint | Input | Cost (credits/sec) | +|-------|----------|-------|---------------------| +| `gen4.5` | `POST /v1/image_to_video` or `POST /v1/text_to_video` | Text and/or Image | 12 | +| `gen4_turbo` | `POST /v1/image_to_video` | Image required | 5 | +| `gen4_aleph` | `POST /v1/video_to_video` | Video + Text/Image | 15 | +| `act_two` | `POST /v1/character_performance` | Image/Video | 5 | +| `veo3` | `POST /v1/image_to_video` or `POST /v1/text_to_video` | Text/Image | 40 | +| `veo3.1` | `POST /v1/image_to_video` or `POST /v1/text_to_video` | Text/Image | 20-40 | +| `veo3.1_fast` | `POST /v1/image_to_video` or `POST /v1/text_to_video` | Text/Image | 10-15 | +| `seedance2` | `POST /v1/text_to_video`, `POST /v1/image_to_video`, or `POST /v1/video_to_video` | Text, Image, and/or Video | 36 | + +Video duration: 2-15 seconds (model-dependent). Aspect ratios are pixel-based: `1280:720`, `720:1280`, `1104:832`, `960:960`, `832:1104`, `1584:672`, etc. + +Seedance 2 specifics: +- Modes: text-to-video, image-to-video (first/last frame or image reference), video-to-video +- Duration: required for TTV and ITV (in seconds) +- Aspect ratios (pixel-based): `1280:720`, `720:1280`, `960:960`, `1112:834`, `834:1112`, `1470:630`, `992:432`, `864:496`, `752:560`, `640:640`, `560:752`, `496:864` +- ITV supports two mutually exclusive modes: first/last frame (`promptImage` array with `position`) or image reference (`references` array) +- VTV input requirements: max 15 seconds, max 32 MB, min 720p resolution, MP4 recommended + +### Image Generation + +| Model | Endpoint | Cost (credits) | +|-------|----------|----------------| +| `gen4_image` | `POST /v1/text_to_image` | 5 (720p), 8 (1080p) | +| `gen4_image_turbo` | `POST /v1/text_to_image` | 2 | +| `gemini_2.5_flash` | `POST /v1/text_to_image` | 5 | + +### Audio Generation + +| Model | Endpoint | Use Case | Cost | +|-------|----------|----------|------| +| `eleven_multilingual_v2` | `POST /v1/text_to_speech` | Text to speech | 1 credit/50 chars | +| `eleven_text_to_sound_v2` | `POST /v1/sound_effect` | Sound effects | 1-2 credits | +| `eleven_voice_isolation` | `POST /v1/voice_isolation` | Isolate voice from audio | 1 credit/6 sec | +| `eleven_voice_dubbing` | `POST /v1/voice_dubbing` | Dub audio to other languages | 1 credit/2 sec | +| `eleven_multilingual_sts_v2` | `POST /v1/speech_to_speech` | Voice conversion | 1 credit/3 sec | + +### Characters (Real-Time Avatars) + +| Model | Description | Session Max Duration | +|-------|-------------|----------------------| +| `gwm1_avatars` | Real-time conversational avatars powered by GWM-1 | 5 minutes | + +Endpoints: + +| Method | Endpoint | Description | +|--------|----------|-------------| +| `POST` | `/v1/avatars` | Create a new Avatar | +| `GET` | `/v1/avatars/{id}` | Retrieve an Avatar | +| `PATCH` | `/v1/avatars/{id}` | Update an Avatar (name, voice, personality, documentIds) | +| `DELETE` | `/v1/avatars/{id}` | Delete an Avatar | +| `POST` | `/v1/realtime_sessions` | Create a new real-time session | +| `GET` | `/v1/realtime_sessions/{id}` | Retrieve session status (poll until `READY`) | +| `POST` | `/v1/realtime_sessions/{id}/consume` | Consume session credentials for WebRTC (one-time use) | + +Avatar creation parameters: + +| Parameter | Type | Description | +|-----------|------|-------------| +| `name` | string | Display name for the avatar | +| `referenceImage` | string | URL or `runway://` URI of the character image | +| `voice` | object | `{ type: 'runway-live-preset', presetId: 'clara' }` | +| `personality` | string | System prompt / personality instructions | +| `documentIds` | string[] | Optional. IDs of knowledge base documents to attach | + +Voice presets: `clara` (soft), `victoria` (firm), `vincent` (authoritative). Preview all at [dev.runwayml.com](https://dev.runwayml.com/). + +Session statuses: `NOT_READY` -> `READY` -> `RUNNING` -> `COMPLETED` (or `FAILED` / `CANCELLED`) + +### Documents (Knowledge Base) + +| Method | Endpoint | Description | +|--------|----------|-------------| +| `POST` | `/v1/documents` | Create a document (plain text or Markdown) | +| `GET` | `/v1/documents/{id}` | Retrieve a document | +| `DELETE` | `/v1/documents/{id}` | Delete a document | + +Each Avatar supports up to 50,000 tokens of knowledge. Link documents to an Avatar via `client.avatars.update(id, { documentIds: [...] })`. + +--- + +## Request Body Reference (raw JSON) + +Use these when calling the API directly (e.g. through `use-runway-api`'s `request` command) rather than via an SDK. Only required + common fields shown - consult `rw-fetch-api-reference` for the full schema. + +### `POST /v1/text_to_image` + +```json +{ + "model": "gen4_image", + "promptText": "A serene Japanese garden with cherry blossoms", + "ratio": "1920:1080" +} +``` + +- `model`: `gen4_image` | `gen4_image_turbo` | `gemini_2.5_flash` (required) +- `promptText`: string, up to ~1000 chars (required) +- `ratio`: one of `1920:1080`, `1080:1920`, `1024:1024`, `1360:768`, `1080:1080`, `1168:880`, `1440:1080`, `1080:1440`, `1808:768`, `2112:912` (required; 720p or 1080p variants depending on model) +- `referenceImages`: optional `[{ "uri": "https://...", "tag": "MyTag" }]` - reference by `@MyTag` in `promptText` +- `seed`: optional integer for reproducibility + +### `POST /v1/text_to_video` + +```json +{ + "model": "gen4.5", + "promptText": "A golden retriever running through wildflowers at sunset", + "ratio": "1280:720", + "duration": 5 +} +``` + +- `model`: `gen4.5` | `veo3` | `veo3.1` | `veo3.1_fast` | `seedance2` (required) +- `duration`: integer seconds, 2-10 (required; model-specific valid values - e.g. veo3 only accepts 8) +- `ratio`: e.g. `1280:720`, `720:1280`, `1104:832`, `832:1104`, `960:960` (required) + +### `POST /v1/image_to_video` + +```json +{ + "model": "gen4.5", + "promptImage": "https://example.com/cover.jpg", + "promptText": "A slow dolly-in shot", + "ratio": "1280:720", + "duration": 5 +} +``` + +- `model`: `gen4.5` | `gen4_turbo` | `veo3` | `veo3.1` | `veo3.1_fast` | `seedance2` (required) +- `promptImage`: HTTPS URL, data URI, or `runway://` URI (required). Can also be `[{ "uri": "...", "position": "first" | "last" }]` for keyframes. +- `promptText`: optional for most models, required for `gen4_turbo` when no image motion is obvious + +### `POST /v1/video_to_video` + +```json +{ + "model": "gen4_aleph", + "videoUri": "https://example.com/source.mp4", + "promptText": "Change the season to winter with snowfall", + "ratio": "1280:720" +} +``` + +### `POST /v1/text_to_speech` + +```json +{ + "model": "eleven_multilingual_v2", + "promptText": "Hello, welcome to Runway.", + "voice": { "type": "runway-preset", "presetId": "Maya" } +} +``` + +- `voice`: `{ type: "runway-preset", presetId: "Maya" | "Noah" | "Leslie" | ... }` or a provider-specific voice object +- `languageCode`: optional ISO code (auto-detected by default) + +### `POST /v1/sound_effect` + +```json +{ + "model": "eleven_text_to_sound_v2", + "promptText": "Thunderclap followed by heavy rain", + "duration": 5 +} +``` + +### `POST /v1/voice_isolation` + +```json +{ + "model": "eleven_voice_isolation", + "audioUri": "https://example.com/noisy.mp3" +} +``` + +### `POST /v1/voice_dubbing` + +```json +{ + "model": "eleven_voice_dubbing", + "audioUri": "https://example.com/english.mp3", + "targetLang": "es" +} +``` + +### `POST /v1/speech_to_speech` + +```json +{ + "model": "eleven_multilingual_sts_v2", + "media": { "type": "audio", "uri": "https://example.com/source.mp3" }, + "voice": { "type": "runway-preset", "presetId": "Maya" } +} +``` + +### `POST /v1/avatars` + +```json +{ + "name": "Support Agent", + "referenceImage": "https://example.com/portrait.jpg", + "voice": { "type": "runway-live-preset", "presetId": "clara" }, + "personality": "You are a friendly support agent.", + "documentIds": [] +} +``` + +### `POST /v1/documents` + +```json +{ + "avatarId": "", + "name": "FAQ", + "content": "Q: What is your return policy?\nA: 30 days, no questions asked." +} +``` + +### `POST /v1/realtime_sessions` + +```json +{ + "avatarId": "" +} +``` + +--- + +### Management Endpoints + +| Method | Endpoint | Description | +|--------|----------|-------------| +| `GET` | `/v1/tasks/{id}` | Get task status and output | +| `DELETE` | `/v1/tasks/{id}` | Cancel/delete a task | +| `POST` | `/v1/uploads` | Create ephemeral upload | +| `GET` | `/v1/organization` | Organization info & credit balance | +| `POST` | `/v1/organization/usage` | Credit usage history (up to 90 days) | + +--- + +## Task Lifecycle + +All generation endpoints return a task object. The flow is: + +1. Submit - `POST /v1/` -> returns `{ "id": "task_xxx" }` +2. Poll - `GET /v1/tasks/{id}` -> returns task with `status` +3. Retrieve output - When `status === "SUCCEEDED"`, the `output` array contains signed URLs + +### Task Statuses + +| Status | Meaning | +|--------|---------| +| `PENDING` | Queued, waiting to start | +| `RUNNING` | Currently generating | +| `SUCCEEDED` | Complete - output URLs available | +| `FAILED` | Generation failed - check `failure` field | +| `THROTTLED` | Concurrency limit hit - auto-queued | + +### SDK Polling (Recommended) + +The SDKs provide a `waitForTaskOutput()` method that handles polling automatically: + +```javascript +// Node.js - polls until complete (default 10 min timeout) +const task = await client.imageToVideo.create({ + model: 'gen4.5', + promptImage: 'https://example.com/image.jpg', + promptText: 'A sunset timelapse', + ratio: '1280:720', + duration: 5 +}).waitForTaskOutput(); + +console.log(task.output); // Array of signed URLs +``` + +```python +# Python +task = client.image_to_video.create( + model='gen4.5', + prompt_image='https://example.com/image.jpg', + prompt_text='A sunset timelapse', + ratio='1280:720', + duration=5 +).wait_for_task_output() + +print(task.output) +``` + +### Manual Polling (REST) + +```javascript +async function pollTask(taskId) { + while (true) { + const response = await fetch(`https://api.dev.runwayml.com/v1/tasks/${taskId}`, { + headers: { + 'Authorization': `Bearer ${process.env.RUNWAYML_API_SECRET}`, + 'X-Runway-Version': '2024-11-06' + } + }); + const task = await response.json(); + + if (task.status === 'SUCCEEDED') return task; + if (task.status === 'FAILED') throw new Error(task.failure); + + await new Promise(r => setTimeout(r, 5000)); // poll every 5 seconds + } +} +``` + +--- + +## Output Handling + +- Successful tasks return an `output` array with signed URLs to generated content +- Output URLs expire within 24-48 hours +- Download and store outputs in your own storage - do not serve signed URLs to end users +- Video outputs are MP4, image outputs are PNG/JPEG + +--- + +## Input Requirements + +### Size Limits + +| Type | Via URL | Via Data URI | Via Upload | +|------|---------|-------------|------------| +| Image | 16 MB | 5 MB | 200 MB | +| Video | 32 MB | 16 MB | 200 MB | +| Audio | 32 MB | 16 MB | 200 MB | + +### Supported Formats + +- Images: JPEG, PNG, WebP (no GIF) +- Video codecs: H.264, H.265/HEVC, AV1, VP8/VP9, Apple ProRes, Theora +- Audio: MP3, AAC, FLAC, PCM, ALAC + +### URL Requirements + +If providing assets via URL: +- HTTPS only (no HTTP) +- Domain names only (no IP addresses) +- No redirects +- Must support HTTP HEAD requests +- Must return valid `Content-Type` and `Content-Length` headers +- Max URL length: 2,048 characters + +--- + +## Rate Limits & Tiers + +| Tier | Concurrency | Daily Gens | Monthly Cap | Unlock | +|------|-------------|------------|-------------|--------| +| 1 (default) | 1-2 | 50-200 | $100 | - | +| 2 | 3 | 500-1,000 | $500 | 1 day + $50 | +| 3 | 5 | 1,000-2,000 | $2,000 | 7 days + $100 | +| 4 | 10 | 5,000-10,000 | $20,000 | 14 days + $1,000 | +| 5 | 20 | 25,000-30,000 | $100,000 | 7 days + $5,000 | + +- No requests-per-minute limit - only daily generation quotas +- Exceeding concurrency -> `THROTTLED` status (auto-queued, not rejected) +- Exceeding daily limit -> `429 Too Many Requests` +- Daily limits use a rolling 24-hour window + +--- + +## Error Handling + +### HTTP Errors + +| Code | Meaning | Action | +|------|---------|--------| +| 400 | Input validation failure | Fix input, do not retry | +| 401 | Invalid API key | Check key, do not retry | +| 429 | Rate limited | Retry with exponential backoff + jitter | +| 502/503/504 | Server overload | Retry with exponential backoff + jitter | + +### Task Failure Codes + +| Code | Meaning | Retry? | +|------|---------|--------| +| `SAFETY.INPUT.*` | Input content moderation | No - not refundable | +| `SAFETY.OUTPUT.*` | Output content moderation | Yes - try different prompt | +| `INTERNAL.BAD_OUTPUT` | Quality issue | Yes | +| `ASSET.INVALID` | Bad input format | Fix input | +| `INTERNAL` | Server error | Yes | + +The SDKs handle retries for transient errors automatically. + +--- + +## Data URI Support + +Base64-encoded images can be passed instead of URLs: + +``` +data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA... +``` + +Useful for small images or when you don't want to host the file. Subject to the data URI size limits above. diff --git a/plugins/runway-api/skills/rw-check-compatibility/SKILL.md b/plugins/runway-api/skills/rw-check-compatibility/SKILL.md new file mode 100644 index 00000000..a613c4e0 --- /dev/null +++ b/plugins/runway-api/skills/rw-check-compatibility/SKILL.md @@ -0,0 +1,116 @@ +--- +name: rw-check-compatibility +description: "Analyze a user's codebase to verify it can use Runway's public API (server-side requirement)" +user-invocable: false +--- + +# Check Compatibility + +Analyze the user's project to determine whether it is compatible with Runway's public API. + +## Why This Matters + +Runway's public API requires server-side invocation. The API key must never be exposed in client-side code. Projects that are purely frontend (static HTML/JS, client-only SPAs without a backend) cannot safely call the API. + +## Analysis Steps + +### Step 1: Identify the Project Type + +Search the project root for these files to determine the stack: + +| File | Indicates | +|------|-----------| +| `package.json` | Node.js project | +| `requirements.txt`, `pyproject.toml`, `Pipfile`, `setup.py` | Python project | +| `go.mod` | Go project | +| `Cargo.toml` | Rust project | +| `pom.xml`, `build.gradle` | Java/Kotlin project | +| `Gemfile` | Ruby project | +| `composer.json` | PHP project | + +If none of these exist, flag the project as unknown and ask the user what language/runtime they're using. + +### Step 2: Check for Server-Side Capability + +Look for indicators of a server/backend: + +Node.js projects - check `package.json` dependencies for: +- `express`, `fastify`, `koa`, `hapi`, `nest`, `hono` -> HTTP server framework +- `next` -> Next.js (has API routes - compatible) +- `nuxt` -> Nuxt.js (has server routes - compatible) +- `remix` -> Remix (has loaders/actions - compatible) +- `@sveltejs/kit` -> SvelteKit (has server routes - compatible) +- `astro` -> Astro (has API endpoints if SSR enabled) + +Python projects - check for: +- `flask`, `django`, `fastapi`, `starlette`, `tornado`, `aiohttp`, `sanic` -> web server framework +- `streamlit`, `gradio` -> can make server-side calls + +Red flags (frontend-only): +- `package.json` with only `react`, `vue`, `svelte`, `angular` and NO server framework +- `vite.config.ts` or `webpack.config.js` with no server/SSR configuration +- Static site generators without server routes (e.g., plain Gatsby, plain Eleventy) +- `index.html` as the only entry point with inline `