diff --git a/compose/agent-zero-compose.yaml b/compose/agent-zero-compose.yaml new file mode 100644 index 00000000..ddea5718 --- /dev/null +++ b/compose/agent-zero-compose.yaml @@ -0,0 +1,77 @@ +# Agent Zero - Autonomous AI Agent Framework +# https://github.com/agent0ai/agent-zero +# +# Agent Zero is an AI framework for personal assistants with features like: +# - Multi-agent cooperation +# - Computer tool usage +# - Memory and knowledge retrieval +# - Custom instruments (tools) + +# ============================================================================= +# USHADOW METADATA (ignored by Docker, read by ushadow backend) +# ============================================================================= +x-ushadow: + agent-zero: + display_name: "Agent Zero" + description: "Autonomous AI agent framework with memory, knowledge, and tool usage capabilities" + requires: [llm] + optional: [] + route_path: /agent-zero + +services: + agent-zero: + image: agent0ai/agent-zero:latest + container_name: ${COMPOSE_PROJECT_NAME:-ushadow}-agent-zero + ports: + - "${AGENT_ZERO_PORT:-50080}:80" + environment: + # LLM Configuration - Agent Zero supports multiple providers + # OpenAI + - OPENAI_API_KEY=${OPENAI_API_KEY:-} + + # Anthropic + - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY:-} + + # Google + - GOOGLE_API_KEY=${GOOGLE_API_KEY:-} + + # Groq + - GROQ_API_KEY=${GROQ_API_KEY:-} + + # OpenRouter + - OPENROUTER_API_KEY=${OPENROUTER_API_KEY:-} + + # Ollama (for local models) + - OLLAMA_BASE_URL=${OLLAMA_BASE_URL:-http://host.docker.internal:11434} + + # Perplexity (for web search) + - PERPLEXITY_API_KEY=${PERPLEXITY_API_KEY:-} + + volumes: + # Main data directory - stores memory, knowledge, instruments, prompts, settings + - agent_zero_data:/a0 + + networks: + - infra-network + + # Enable access to host network for Ollama and other local services + extra_hosts: + - "host.docker.internal:host-gateway" + + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:80/"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 30s + + restart: unless-stopped + +volumes: + agent_zero_data: + name: ${COMPOSE_PROJECT_NAME:-ushadow}_agent_zero_data + +networks: + infra-network: + external: true + name: ${COMPOSE_PROJECT_NAME:-ushadow}_infra-network diff --git a/ushadow/backend/main.py b/ushadow/backend/main.py index bbab13e7..118b4912 100644 --- a/ushadow/backend/main.py +++ b/ushadow/backend/main.py @@ -22,6 +22,7 @@ from src.routers import health, wizard, chronicle, auth, feature_flags from src.routers import services, deployments, providers, instances, chat from src.routers import kubernetes, tailscale, unodes, docker +from src.routers import github_import from src.routers import settings as settings_api from src.middleware import setup_middleware from src.services.unode_manager import init_unode_manager, get_unode_manager @@ -180,6 +181,7 @@ def send_telemetry(): app.include_router(chat.router, prefix="/api/chat", tags=["chat"]) app.include_router(deployments.router, tags=["deployments"]) app.include_router(tailscale.router, tags=["tailscale"]) +app.include_router(github_import.router, prefix="/api/github-import", tags=["github-import"]) # Setup MCP server for LLM tool access setup_mcp_server(app) diff --git a/ushadow/backend/src/models/github_import.py b/ushadow/backend/src/models/github_import.py new file mode 100644 index 00000000..08947751 --- /dev/null +++ b/ushadow/backend/src/models/github_import.py @@ -0,0 +1,406 @@ +""" +Models for Docker Import functionality. + +Supports importing docker-compose files from: +- GitHub repositories +- Docker Hub images + +And configuring shadow headers and environment variables. +""" + +from typing import Dict, List, Optional, Any, Literal +from pydantic import BaseModel, Field, field_validator +import re + + +# ============================================================================= +# Import Source Types +# ============================================================================= + +ImportSourceType = Literal["github", "dockerhub"] + + +class DockerHubImageInfo(BaseModel): + """Parsed Docker Hub URL information.""" + namespace: str # e.g., "fishaudio" or "library" for official images + repository: str # e.g., "fish-speech" + tag: str = "latest" + + @property + def full_image_name(self) -> str: + """Get full Docker image name.""" + if self.namespace == "library": + # Official images don't need namespace prefix + return f"{self.repository}:{self.tag}" + return f"{self.namespace}/{self.repository}:{self.tag}" + + @property + def api_url(self) -> str: + """Get Docker Hub API URL for image info.""" + return f"https://hub.docker.com/v2/repositories/{self.namespace}/{self.repository}" + + @property + def tags_url(self) -> str: + """Get Docker Hub API URL for tags.""" + return f"https://hub.docker.com/v2/repositories/{self.namespace}/{self.repository}/tags" + + +class GitHubUrlInfo(BaseModel): + """Parsed GitHub URL information.""" + owner: str + repo: str + branch: str = "main" + path: str = "" + + @property + def api_url(self) -> str: + """Get GitHub API URL for contents.""" + if self.path: + return f"https://api.github.com/repos/{self.owner}/{self.repo}/contents/{self.path}?ref={self.branch}" + return f"https://api.github.com/repos/{self.owner}/{self.repo}/contents?ref={self.branch}" + + @property + def raw_url(self) -> str: + """Get raw GitHub URL for file content.""" + if self.path: + return f"https://raw.githubusercontent.com/{self.owner}/{self.repo}/{self.branch}/{self.path}" + return f"https://raw.githubusercontent.com/{self.owner}/{self.repo}/{self.branch}" + + +class GitHubImportRequest(BaseModel): + """Request to import from a GitHub URL.""" + github_url: str = Field(..., description="GitHub repository or file URL") + branch: Optional[str] = Field(None, description="Branch to use (defaults to main/master)") + compose_path: Optional[str] = Field(None, description="Path to docker-compose file if not auto-detected") + + @field_validator('github_url') + @classmethod + def validate_github_url(cls, v: str) -> str: + """Validate that the URL is a GitHub URL.""" + if not v: + raise ValueError("GitHub URL is required") + if not ('github.com' in v or 'raw.githubusercontent.com' in v): + raise ValueError("URL must be a GitHub URL") + return v + + +class DetectedComposeFile(BaseModel): + """A docker-compose file detected in the repository.""" + path: str + name: str + download_url: str + size: int = 0 + + +class ComposeEnvVarInfo(BaseModel): + """Environment variable extracted from compose file.""" + name: str + has_default: bool = False + default_value: Optional[str] = None + is_required: bool = True + description: Optional[str] = None + + +class ComposeServiceInfo(BaseModel): + """Service information extracted from compose file.""" + name: str + image: Optional[str] = None + ports: List[Dict[str, Any]] = Field(default_factory=list) + environment: List[ComposeEnvVarInfo] = Field(default_factory=list) + depends_on: List[str] = Field(default_factory=list) + volumes: List[str] = Field(default_factory=list) + networks: List[str] = Field(default_factory=list) + command: Optional[str] = None + healthcheck: Optional[Dict[str, Any]] = None + + +class ShadowHeaderConfig(BaseModel): + """Configuration for shadow header routing.""" + enabled: bool = True + header_name: str = Field(default="X-Shadow-Service", description="Header name for service identification") + header_value: Optional[str] = Field(None, description="Header value (defaults to service name)") + route_path: Optional[str] = Field(None, description="Tailscale Serve route path (e.g., /myservice)") + + +class EnvVarConfigItem(BaseModel): + """Configuration for a single environment variable.""" + name: str + source: str = "literal" # "literal", "setting", "default" + value: Optional[str] = None + setting_path: Optional[str] = None + is_secret: bool = False + + +class PortConfig(BaseModel): + """Configuration for a port mapping.""" + host_port: int + container_port: int + protocol: str = "tcp" + + +class VolumeConfig(BaseModel): + """Configuration for a volume mount.""" + name: str + container_path: str + is_named_volume: bool = True # True for named volumes, False for bind mounts + + +class ImportedServiceConfig(BaseModel): + """Full configuration for an imported service.""" + service_name: str + display_name: Optional[str] = None + description: Optional[str] = None + # Source can be GitHub or Docker Hub + source_type: ImportSourceType = "github" + source_url: str # GitHub URL or Docker Hub URL + # For GitHub imports + compose_path: Optional[str] = None + # For Docker Hub imports + docker_image: Optional[str] = None + ports: List[PortConfig] = Field(default_factory=list) + volumes: List[VolumeConfig] = Field(default_factory=list) + # Common config + shadow_header: ShadowHeaderConfig = Field(default_factory=ShadowHeaderConfig) + env_vars: List[EnvVarConfigItem] = Field(default_factory=list) + enabled: bool = True + # Capabilities this service provides (e.g., ['llm', 'tts']) + capabilities: List[str] = Field(default_factory=list, description="Capabilities this service provides") + + # Backwards compatibility + @property + def github_url(self) -> str: + return self.source_url + + +class GitHubScanResponse(BaseModel): + """Response from scanning a GitHub repository.""" + success: bool + github_info: Optional[GitHubUrlInfo] = None + compose_files: List[DetectedComposeFile] = Field(default_factory=list) + message: Optional[str] = None + error: Optional[str] = None + + +class ComposeParseResponse(BaseModel): + """Response from parsing a docker-compose file.""" + success: bool + compose_path: str + services: List[ComposeServiceInfo] = Field(default_factory=list) + networks: List[str] = Field(default_factory=list) + volumes: List[str] = Field(default_factory=list) + message: Optional[str] = None + error: Optional[str] = None + + +class ImportServiceRequest(BaseModel): + """Request to import and register a service from GitHub.""" + github_url: str + compose_path: str + service_name: str + config: ImportedServiceConfig + + +class ImportServiceResponse(BaseModel): + """Response from importing a service.""" + success: bool + service_id: Optional[str] = None + service_name: Optional[str] = None + message: str + compose_file_path: Optional[str] = None + + +def parse_github_url(url: str) -> GitHubUrlInfo: + """ + Parse a GitHub URL into its components. + + Supports: + - https://github.com/owner/repo + - https://github.com/owner/repo/tree/branch + - https://github.com/owner/repo/tree/branch/path/to/dir + - https://github.com/owner/repo/blob/branch/path/to/file + - https://raw.githubusercontent.com/owner/repo/branch/path/to/file + """ + url = url.strip() + + # Handle raw.githubusercontent.com URLs + raw_match = re.match( + r'https?://raw\.githubusercontent\.com/([^/]+)/([^/]+)/([^/]+)(?:/(.+))?', + url + ) + if raw_match: + owner, repo, branch, path = raw_match.groups() + return GitHubUrlInfo( + owner=owner, + repo=repo, + branch=branch, + path=path or "" + ) + + # Handle github.com URLs + github_match = re.match( + r'https?://github\.com/([^/]+)/([^/]+)(?:/(tree|blob)/([^/]+)(?:/(.+))?)?(?:\.git)?', + url + ) + if github_match: + owner, repo, url_type, branch, path = github_match.groups() + # Remove .git suffix if present + repo = repo.replace('.git', '') + return GitHubUrlInfo( + owner=owner, + repo=repo, + branch=branch or "main", + path=path or "" + ) + + # Simple github.com/owner/repo pattern + simple_match = re.match( + r'https?://github\.com/([^/]+)/([^/]+)/?$', + url + ) + if simple_match: + owner, repo = simple_match.groups() + repo = repo.replace('.git', '') + return GitHubUrlInfo( + owner=owner, + repo=repo, + branch="main", + path="" + ) + + raise ValueError(f"Could not parse GitHub URL: {url}") + + +def parse_dockerhub_url(url: str) -> DockerHubImageInfo: + """ + Parse a Docker Hub URL into its components. + + Supports: + - https://hub.docker.com/r/namespace/repository + - https://hub.docker.com/r/namespace/repository/tags + - https://hub.docker.com/_/official-image (official images) + - namespace/repository (direct image reference) + - namespace/repository:tag + """ + url = url.strip() + + # Handle hub.docker.com URLs + dockerhub_match = re.match( + r'https?://hub\.docker\.com/r/([^/]+)/([^/]+)(?:/tags)?/?$', + url + ) + if dockerhub_match: + namespace, repository = dockerhub_match.groups() + return DockerHubImageInfo( + namespace=namespace, + repository=repository, + tag="latest" + ) + + # Handle official images: https://hub.docker.com/_/image-name + official_match = re.match( + r'https?://hub\.docker\.com/_/([^/]+)/?$', + url + ) + if official_match: + repository = official_match.group(1) + return DockerHubImageInfo( + namespace="library", + repository=repository, + tag="latest" + ) + + # Handle direct image reference: namespace/repository:tag + direct_match = re.match( + r'^([^/:]+)/([^/:]+)(?::([^/]+))?$', + url + ) + if direct_match: + namespace, repository, tag = direct_match.groups() + return DockerHubImageInfo( + namespace=namespace, + repository=repository, + tag=tag or "latest" + ) + + # Handle official image direct reference: image-name:tag + official_direct_match = re.match( + r'^([^/:]+)(?::([^/]+))?$', + url + ) + if official_direct_match: + repository, tag = official_direct_match.groups() + return DockerHubImageInfo( + namespace="library", + repository=repository, + tag=tag or "latest" + ) + + raise ValueError(f"Could not parse Docker Hub URL: {url}") + + +def detect_import_source(url: str) -> ImportSourceType: + """Detect whether a URL is GitHub or Docker Hub.""" + url = url.strip().lower() + if 'github.com' in url or 'raw.githubusercontent.com' in url: + return "github" + if 'hub.docker.com' in url or 'docker.io' in url: + return "dockerhub" + # Check if it looks like a docker image reference (namespace/repo or repo:tag) + if re.match(r'^[a-z0-9_-]+/[a-z0-9_-]+(?::[a-z0-9._-]+)?$', url): + return "dockerhub" + if re.match(r'^[a-z0-9_-]+(?::[a-z0-9._-]+)?$', url): + return "dockerhub" + # Default to github for URLs + if url.startswith('http'): + return "github" + return "dockerhub" + + +class DockerHubScanResponse(BaseModel): + """Response from scanning a Docker Hub image.""" + success: bool + image_info: Optional[DockerHubImageInfo] = None + description: Optional[str] = None + stars: int = 0 + pulls: int = 0 + available_tags: List[str] = Field(default_factory=list) + message: Optional[str] = None + error: Optional[str] = None + + +class DockerHubImportRequest(BaseModel): + """Request to import from Docker Hub.""" + dockerhub_url: str = Field(..., description="Docker Hub URL or image reference") + tag: Optional[str] = Field(None, description="Image tag (defaults to latest)") + + @field_validator('dockerhub_url') + @classmethod + def validate_dockerhub_url(cls, v: str) -> str: + """Validate the Docker Hub URL or image reference.""" + if not v: + raise ValueError("Docker Hub URL or image reference is required") + return v + + +class UnifiedImportRequest(BaseModel): + """Unified request for importing from any supported source.""" + url: str = Field(..., description="GitHub URL, Docker Hub URL, or image reference") + branch: Optional[str] = Field(None, description="Branch for GitHub (defaults to main)") + tag: Optional[str] = Field(None, description="Tag for Docker Hub (defaults to latest)") + compose_path: Optional[str] = Field(None, description="Path to docker-compose file if not auto-detected") + + +class UnifiedScanResponse(BaseModel): + """Unified response from scanning any import source.""" + success: bool + source_type: ImportSourceType + # GitHub-specific + github_info: Optional[GitHubUrlInfo] = None + compose_files: List[DetectedComposeFile] = Field(default_factory=list) + # Docker Hub-specific + dockerhub_info: Optional[DockerHubImageInfo] = None + available_tags: List[str] = Field(default_factory=list) + image_description: Optional[str] = None + # Common + message: Optional[str] = None + error: Optional[str] = None diff --git a/ushadow/backend/src/routers/github_import.py b/ushadow/backend/src/routers/github_import.py new file mode 100644 index 00000000..78a5055f --- /dev/null +++ b/ushadow/backend/src/routers/github_import.py @@ -0,0 +1,1130 @@ +""" +Docker Import Router - GitHub and Docker Hub. + +Provides endpoints for: +- Scanning GitHub repositories for docker-compose files +- Scanning Docker Hub for image information +- Parsing compose files and extracting service/env info +- Creating compose files from Docker Hub images +- Importing and registering services with shadow header configuration +""" + +import logging +import os +import re +from pathlib import Path +from typing import List, Dict, Any, Optional + +import httpx +from fastapi import APIRouter, HTTPException, Depends +from ruamel.yaml import YAML + +from src.models.github_import import ( + GitHubImportRequest, + GitHubScanResponse, + GitHubUrlInfo, + DetectedComposeFile, + ComposeParseResponse, + ComposeServiceInfo, + ComposeEnvVarInfo, + ImportServiceRequest, + ImportServiceResponse, + ImportedServiceConfig, + ShadowHeaderConfig, + EnvVarConfigItem, + PortConfig, + VolumeConfig, + parse_github_url, + # Docker Hub imports + DockerHubImageInfo, + DockerHubScanResponse, + DockerHubImportRequest, + UnifiedImportRequest, + UnifiedScanResponse, + parse_dockerhub_url, + detect_import_source, +) +from src.services.auth import get_current_user +from src.models.user import User +from src.config.yaml_parser import ComposeParser + +logger = logging.getLogger(__name__) +router = APIRouter() + +# Common docker-compose file patterns to look for +COMPOSE_FILE_PATTERNS = [ + 'docker-compose.yml', + 'docker-compose.yaml', + 'compose.yml', + 'compose.yaml', + 'docker-compose.dev.yml', + 'docker-compose.dev.yaml', + 'docker-compose.prod.yml', + 'docker-compose.prod.yaml', + 'docker-compose.override.yml', + 'docker-compose.override.yaml', +] + + +async def fetch_github_contents(url: str, token: Optional[str] = None) -> Dict[str, Any]: + """Fetch contents from GitHub API.""" + headers = { + 'Accept': 'application/vnd.github.v3+json', + 'User-Agent': 'Ushadow-Import' + } + if token: + headers['Authorization'] = f'token {token}' + + async with httpx.AsyncClient(timeout=30.0) as client: + response = await client.get(url, headers=headers) + if response.status_code == 404: + raise HTTPException(status_code=404, detail="Repository or path not found") + if response.status_code == 403: + raise HTTPException(status_code=403, detail="GitHub API rate limit exceeded or access denied") + response.raise_for_status() + return response.json() + + +async def fetch_raw_content(url: str, token: Optional[str] = None) -> str: + """Fetch raw file content from GitHub.""" + headers = {'User-Agent': 'Ushadow-Import'} + if token: + headers['Authorization'] = f'token {token}' + + async with httpx.AsyncClient(timeout=30.0) as client: + response = await client.get(url, headers=headers) + response.raise_for_status() + return response.text + + +def detect_compose_files(contents: List[Dict[str, Any]]) -> List[DetectedComposeFile]: + """Detect docker-compose files from GitHub directory listing.""" + compose_files = [] + + for item in contents: + if item.get('type') != 'file': + continue + + name = item.get('name', '').lower() + + # Check if file matches any compose pattern + for pattern in COMPOSE_FILE_PATTERNS: + if name == pattern.lower() or name.endswith('-compose.yml') or name.endswith('-compose.yaml'): + compose_files.append(DetectedComposeFile( + path=item.get('path', ''), + name=item.get('name', ''), + download_url=item.get('download_url', ''), + size=item.get('size', 0) + )) + break + + return compose_files + + +def parse_env_var_line(line: str) -> Optional[ComposeEnvVarInfo]: + """Parse an environment variable line from compose format.""" + if not line or not isinstance(line, str): + return None + + line = line.strip() + if not line: + return None + + # Pattern for ${VAR:-default} + env_pattern = re.compile(r'\$\{([^:}]+)(?::-([^}]*))?\}') + + # Check for KEY=VALUE format + if '=' in line: + key, value = line.split('=', 1) + key = key.strip() + value = value.strip() + + if not value: + return ComposeEnvVarInfo(name=key, has_default=False, is_required=True) + + # Check for ${VAR:-default} pattern + match = env_pattern.search(value) + if match: + var_name, default = match.groups() + has_default = default is not None and default != "" + return ComposeEnvVarInfo( + name=key, + has_default=has_default, + default_value=default if has_default else None, + is_required=not has_default + ) + + # Plain value - has a hardcoded default + return ComposeEnvVarInfo( + name=key, + has_default=True, + default_value=value, + is_required=False + ) + + # Bare variable name - required + return ComposeEnvVarInfo(name=line, has_default=False, is_required=True) + + +def parse_compose_content(content: str) -> Dict[str, Any]: + """Parse docker-compose YAML content.""" + yaml = YAML() + yaml.preserve_quotes = True + from io import StringIO + return yaml.load(StringIO(content)) or {} + + +def extract_services_from_compose(data: Dict[str, Any]) -> List[ComposeServiceInfo]: + """Extract service information from parsed compose data.""" + services = [] + services_data = data.get('services', {}) + + for name, service_data in services_data.items(): + # Parse environment variables + env_vars = [] + environment = service_data.get('environment', []) + + if isinstance(environment, list): + for item in environment: + env_info = parse_env_var_line(str(item) if item else "") + if env_info: + env_vars.append(env_info) + elif isinstance(environment, dict): + for key, value in environment.items(): + if value is None: + env_vars.append(ComposeEnvVarInfo(name=key, has_default=False, is_required=True)) + else: + env_info = parse_env_var_line(f"{key}={value}") + if env_info: + env_vars.append(env_info) + + # Parse ports + ports = [] + for port in service_data.get('ports', []): + if isinstance(port, str): + port_str = port.replace('/tcp', '').replace('/udp', '') + if ':' in port_str: + host, container = port_str.rsplit(':', 1) + ports.append({'host': host, 'container': container}) + else: + ports.append({'container': port_str}) + elif isinstance(port, dict): + ports.append({ + 'host': str(port.get('published', '')), + 'container': str(port.get('target', '')) + }) + + # Parse depends_on + depends_on = service_data.get('depends_on', []) + if isinstance(depends_on, dict): + depends_on = list(depends_on.keys()) + + # Parse volumes + volumes = service_data.get('volumes', []) + if isinstance(volumes, list): + volumes = [str(v) for v in volumes] + + # Parse networks + networks = service_data.get('networks', []) + if isinstance(networks, dict): + networks = list(networks.keys()) + + services.append(ComposeServiceInfo( + name=name, + image=service_data.get('image'), + ports=ports, + environment=env_vars, + depends_on=depends_on, + volumes=volumes, + networks=networks, + command=service_data.get('command'), + healthcheck=service_data.get('healthcheck') + )) + + return services + + +# ============================================================================= +# Docker Hub Functions +# ============================================================================= + +async def fetch_dockerhub_info(image_info: DockerHubImageInfo) -> Dict[str, Any]: + """Fetch image information from Docker Hub API.""" + headers = { + 'Accept': 'application/json', + 'User-Agent': 'Ushadow-Import' + } + + async with httpx.AsyncClient(timeout=30.0) as client: + # Fetch repository info + response = await client.get(image_info.api_url, headers=headers) + if response.status_code == 404: + raise HTTPException(status_code=404, detail="Docker Hub image not found") + response.raise_for_status() + return response.json() + + +async def fetch_dockerhub_tags(image_info: DockerHubImageInfo, limit: int = 25) -> List[str]: + """Fetch available tags for a Docker Hub image.""" + headers = { + 'Accept': 'application/json', + 'User-Agent': 'Ushadow-Import' + } + + async with httpx.AsyncClient(timeout=30.0) as client: + url = f"{image_info.tags_url}?page_size={limit}" + response = await client.get(url, headers=headers) + if response.status_code != 200: + return ["latest"] + + data = response.json() + tags = [tag.get('name', 'latest') for tag in data.get('results', [])] + return tags if tags else ["latest"] + + +def generate_compose_from_dockerhub( + image_info: DockerHubImageInfo, + service_name: str, + ports: List[PortConfig], + volumes: List[VolumeConfig], + env_vars: List[EnvVarConfigItem], + shadow_header: ShadowHeaderConfig, + display_name: Optional[str] = None, + description: Optional[str] = None, + capabilities: Optional[List[str]] = None, +) -> str: + """Generate a docker-compose.yaml from Docker Hub image info.""" + yaml = YAML() + yaml.default_flow_style = False + + # Build compose structure + service_metadata = { + 'display_name': display_name or service_name.replace('-', ' ').title(), + 'description': description or f"Imported from Docker Hub: {image_info.full_image_name}", + 'requires': [], + 'optional': [], + 'dockerhub_source': { + 'namespace': image_info.namespace, + 'repository': image_info.repository, + 'tag': image_info.tag, + 'full_image': image_info.full_image_name + } + } + + # Add capabilities if provided + if capabilities and len(capabilities) > 0: + if len(capabilities) == 1: + service_metadata['provides'] = capabilities[0] + else: + service_metadata['provides'] = capabilities + + compose_data = { + 'x-ushadow': { + service_name: service_metadata + }, + 'services': { + service_name: { + 'image': image_info.full_image_name, + 'container_name': f'${{COMPOSE_PROJECT_NAME:-ushadow}}-{service_name}', + } + }, + 'networks': { + 'infra-network': { + 'external': True, + 'name': '${COMPOSE_PROJECT_NAME:-ushadow}_infra-network' + } + } + } + + service_config = compose_data['services'][service_name] + + # Add shadow header config + if shadow_header.enabled: + compose_data['x-ushadow'][service_name]['shadow_header'] = { + 'enabled': True, + 'header_name': shadow_header.header_name, + 'header_value': shadow_header.header_value or service_name + } + if shadow_header.route_path: + compose_data['x-ushadow'][service_name]['route_path'] = shadow_header.route_path + + # Add ports + if ports: + service_config['ports'] = [ + f"${{{service_name.upper().replace('-', '_')}_PORT:-{p.host_port}}}:{p.container_port}" + for p in ports + ] + + # Add environment variables + if env_vars: + env_list = [] + for ev in env_vars: + if ev.source == 'literal' and ev.value: + env_list.append(f"{ev.name}=${{{ev.name}:-{ev.value}}}") + else: + env_list.append(f"{ev.name}=${{{ev.name}:-}}") + if env_list: + service_config['environment'] = env_list + + # Add volumes + if volumes: + service_config['volumes'] = [] + volume_definitions = {} + for v in volumes: + if v.is_named_volume: + volume_name = f"{service_name}_{v.name}" + service_config['volumes'].append(f"{volume_name}:{v.container_path}") + volume_definitions[volume_name] = { + 'name': f'${{COMPOSE_PROJECT_NAME:-ushadow}}_{volume_name}' + } + else: + service_config['volumes'].append(f"{v.name}:{v.container_path}") + + if volume_definitions: + compose_data['volumes'] = volume_definitions + + # Add network + service_config['networks'] = ['infra-network'] + + # Add extra_hosts for host.docker.internal + service_config['extra_hosts'] = ['host.docker.internal:host-gateway'] + + # Add healthcheck placeholder + service_config['healthcheck'] = { + 'test': ['CMD', 'echo', 'ok'], + 'interval': '30s', + 'timeout': '10s', + 'retries': 3, + 'start_period': '30s' + } + + # Add restart policy + service_config['restart'] = 'unless-stopped' + + # Serialize to YAML + from io import StringIO + output = StringIO() + yaml.dump(compose_data, output) + return output.getvalue() + + +# ============================================================================= +# GitHub Endpoints +# ============================================================================= + +@router.post("/scan", response_model=GitHubScanResponse) +async def scan_github_repo( + request: GitHubImportRequest, + current_user: User = Depends(get_current_user) +) -> GitHubScanResponse: + """ + Scan a GitHub repository for docker-compose files. + + Accepts a GitHub URL (repository, directory, or specific file) and returns + a list of detected docker-compose files. + """ + try: + github_info = parse_github_url(request.github_url) + logger.info(f"Scanning GitHub repo: {github_info.owner}/{github_info.repo}") + + # If a specific compose path is provided, verify it exists + if request.compose_path: + github_info.path = request.compose_path + + # Fetch directory contents + contents = await fetch_github_contents(github_info.api_url) + + # Handle single file case + if isinstance(contents, dict) and contents.get('type') == 'file': + return GitHubScanResponse( + success=True, + github_info=github_info, + compose_files=[DetectedComposeFile( + path=contents.get('path', ''), + name=contents.get('name', ''), + download_url=contents.get('download_url', ''), + size=contents.get('size', 0) + )], + message="Found specified compose file" + ) + + # Detect compose files in directory + compose_files = detect_compose_files(contents if isinstance(contents, list) else []) + + if not compose_files: + return GitHubScanResponse( + success=True, + github_info=github_info, + compose_files=[], + message="No docker-compose files found in the specified location" + ) + + return GitHubScanResponse( + success=True, + github_info=github_info, + compose_files=compose_files, + message=f"Found {len(compose_files)} docker-compose file(s)" + ) + + except ValueError as e: + return GitHubScanResponse( + success=False, + error=str(e) + ) + except HTTPException: + raise + except Exception as e: + logger.error(f"Error scanning GitHub repo: {e}") + return GitHubScanResponse( + success=False, + error=str(e) + ) + + +@router.post("/parse", response_model=ComposeParseResponse) +async def parse_compose_file( + github_url: str, + compose_path: str, + current_user: User = Depends(get_current_user) +) -> ComposeParseResponse: + """ + Parse a docker-compose file and extract service/environment information. + + Returns structured information about services, including environment variables + that need to be configured. + """ + try: + github_info = parse_github_url(github_url) + github_info.path = compose_path + + # Build raw URL for file content + raw_url = f"https://raw.githubusercontent.com/{github_info.owner}/{github_info.repo}/{github_info.branch}/{compose_path}" + + # Fetch compose file content + content = await fetch_raw_content(raw_url) + + # Parse the compose file + data = parse_compose_content(content) + + # Extract services + services = extract_services_from_compose(data) + + # Extract networks + networks = list(data.get('networks', {}).keys()) + + # Extract volumes + volumes = list(data.get('volumes', {}).keys()) + + return ComposeParseResponse( + success=True, + compose_path=compose_path, + services=services, + networks=networks, + volumes=volumes, + message=f"Successfully parsed {len(services)} service(s)" + ) + + except Exception as e: + logger.error(f"Error parsing compose file: {e}") + return ComposeParseResponse( + success=False, + compose_path=compose_path, + error=str(e) + ) + + +@router.post("/register", response_model=ImportServiceResponse) +async def register_imported_service( + request: ImportServiceRequest, + current_user: User = Depends(get_current_user) +) -> ImportServiceResponse: + """ + Register an imported service from GitHub. + + Downloads the compose file, saves it locally, and registers the service + with the configured shadow header and environment variables. + """ + try: + github_info = parse_github_url(request.github_url) + github_info.path = request.compose_path + config = request.config + + # Build raw URL and fetch compose content + raw_url = f"https://raw.githubusercontent.com/{github_info.owner}/{github_info.repo}/{github_info.branch}/{request.compose_path}" + content = await fetch_raw_content(raw_url) + + # Parse to validate + data = parse_compose_content(content) + if not data.get('services'): + return ImportServiceResponse( + success=False, + message="No services found in compose file" + ) + + # Add x-ushadow metadata to the compose content + yaml = YAML() + yaml.preserve_quotes = True + from io import StringIO + compose_data = yaml.load(StringIO(content)) or {} + + # Build x-ushadow section + x_ushadow = compose_data.get('x-ushadow', {}) + + # Add metadata for the service + service_meta = { + 'display_name': config.display_name or request.service_name, + 'description': config.description or f"Imported from {github_info.owner}/{github_info.repo}", + 'requires': [], + 'github_source': { + 'url': request.github_url, + 'owner': github_info.owner, + 'repo': github_info.repo, + 'branch': github_info.branch, + 'path': request.compose_path + } + } + + # Add capabilities if provided + if config.capabilities and len(config.capabilities) > 0: + if len(config.capabilities) == 1: + service_meta['provides'] = config.capabilities[0] + else: + service_meta['provides'] = config.capabilities + + # Add shadow header config + if config.shadow_header.enabled: + service_meta['shadow_header'] = { + 'enabled': True, + 'header_name': config.shadow_header.header_name, + 'header_value': config.shadow_header.header_value or request.service_name + } + if config.shadow_header.route_path: + service_meta['route_path'] = config.shadow_header.route_path + + x_ushadow[request.service_name] = service_meta + compose_data['x-ushadow'] = x_ushadow + + # Ensure compose directory exists + compose_dir = Path('/compose') + if not compose_dir.exists(): + compose_dir = Path('compose') + compose_dir.mkdir(parents=True, exist_ok=True) + + # Generate filename + safe_name = re.sub(r'[^a-zA-Z0-9_-]', '-', request.service_name) + compose_filename = f"{safe_name}-compose.yaml" + compose_path = compose_dir / compose_filename + + # Write the compose file with modifications + output = StringIO() + yaml.dump(compose_data, output) + compose_content = output.getvalue() + + # Add environment variable overrides if configured + env_file_content = [] + for env_config in config.env_vars: + if env_config.source == "literal" and env_config.value: + env_file_content.append(f"{env_config.name}={env_config.value}") + + # Write compose file + with open(compose_path, 'w') as f: + f.write(compose_content) + + # Write env file if we have overrides + if env_file_content: + env_file_path = compose_dir / f"{safe_name}.env" + with open(env_file_path, 'w') as f: + f.write('\n'.join(env_file_content) + '\n') + + # Save service configuration to settings + from src.config.omegaconf_settings import get_settings_store + settings = get_settings_store() + + service_config_key = f"imported_services.{safe_name}" + await settings.update({ + service_config_key: { + 'github_url': request.github_url, + 'compose_path': request.compose_path, + 'compose_file': str(compose_path), + 'service_name': request.service_name, + 'display_name': config.display_name, + 'description': config.description, + 'shadow_header': config.shadow_header.model_dump(), + 'env_vars': [ev.model_dump() for ev in config.env_vars], + 'enabled': config.enabled + } + }) + + # Refresh compose registry to pick up new service + from src.services.compose_registry import get_compose_registry + registry = get_compose_registry() + registry.refresh() + + # Auto-install the service (mark as added so it shows in installed services) + installed_key = f"installed_services.{request.service_name}" + await settings.update({ + f"{installed_key}.added": True, + f"{installed_key}.enabled": True + }) + + logger.info(f"Imported and installed service '{request.service_name}' from GitHub: {request.github_url}") + + return ImportServiceResponse( + success=True, + service_id=f"{compose_filename.replace('.yaml', '')}:{request.service_name}", + service_name=request.service_name, + message=f"Successfully imported service '{request.service_name}'", + compose_file_path=str(compose_path) + ) + + except Exception as e: + logger.error(f"Error registering imported service: {e}") + return ImportServiceResponse( + success=False, + message=f"Failed to import service: {str(e)}" + ) + + +@router.get("/imported") +async def list_imported_services( + current_user: User = Depends(get_current_user) +) -> List[Dict[str, Any]]: + """List all imported services from GitHub.""" + try: + from src.config.omegaconf_settings import get_settings_store + settings = get_settings_store() + + imported = settings.get("imported_services", {}) + return [ + { + 'id': key, + **value + } + for key, value in imported.items() + ] if isinstance(imported, dict) else [] + + except Exception as e: + logger.error(f"Error listing imported services: {e}") + return [] + + +@router.delete("/imported/{service_id}") +async def delete_imported_service( + service_id: str, + current_user: User = Depends(get_current_user) +) -> Dict[str, Any]: + """Delete an imported service.""" + try: + from src.config.omegaconf_settings import get_settings_store + settings = get_settings_store() + + imported = settings.get("imported_services", {}) + if service_id not in imported: + raise HTTPException(status_code=404, detail=f"Imported service '{service_id}' not found") + + service_config = imported[service_id] + compose_file = service_config.get('compose_file') + + # Remove compose file if it exists + if compose_file and os.path.exists(compose_file): + os.remove(compose_file) + logger.info(f"Removed compose file: {compose_file}") + + # Remove env file if it exists + env_file = compose_file.replace('-compose.yaml', '.env') if compose_file else None + if env_file and os.path.exists(env_file): + os.remove(env_file) + + # Remove from settings + del imported[service_id] + await settings.update({"imported_services": imported}) + + # Refresh compose registry + from src.services.compose_registry import get_compose_registry + registry = get_compose_registry() + registry.refresh() + + return { + "success": True, + "message": f"Deleted imported service '{service_id}'" + } + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error deleting imported service: {e}") + raise HTTPException(status_code=500, detail=str(e)) + + +@router.put("/imported/{service_id}/config") +async def update_imported_service_config( + service_id: str, + config: ImportedServiceConfig, + current_user: User = Depends(get_current_user) +) -> Dict[str, Any]: + """Update configuration for an imported service.""" + try: + from src.config.omegaconf_settings import get_settings_store + settings = get_settings_store() + + imported = settings.get("imported_services", {}) + if service_id not in imported: + raise HTTPException(status_code=404, detail=f"Imported service '{service_id}' not found") + + # Update configuration + imported[service_id].update({ + 'display_name': config.display_name, + 'description': config.description, + 'shadow_header': config.shadow_header.model_dump(), + 'env_vars': [ev.model_dump() for ev in config.env_vars], + 'enabled': config.enabled + }) + + await settings.update({"imported_services": imported}) + + # Update env file with new values + compose_file = imported[service_id].get('compose_file') + if compose_file: + env_file_content = [] + for env_config in config.env_vars: + if env_config.source == "literal" and env_config.value: + env_file_content.append(f"{env_config.name}={env_config.value}") + + if env_file_content: + env_file_path = compose_file.replace('-compose.yaml', '.env') + with open(env_file_path, 'w') as f: + f.write('\n'.join(env_file_content) + '\n') + + return { + "success": True, + "message": f"Updated configuration for '{service_id}'" + } + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error updating imported service config: {e}") + raise HTTPException(status_code=500, detail=str(e)) + + +# ============================================================================= +# Docker Hub Endpoints +# ============================================================================= + +@router.post("/dockerhub/scan", response_model=DockerHubScanResponse) +async def scan_dockerhub_image( + request: DockerHubImportRequest, + current_user: User = Depends(get_current_user) +) -> DockerHubScanResponse: + """ + Scan a Docker Hub image for information. + + Accepts a Docker Hub URL or image reference and returns + image details and available tags. + """ + try: + image_info = parse_dockerhub_url(request.dockerhub_url) + + # Override tag if specified + if request.tag: + image_info.tag = request.tag + + logger.info(f"Scanning Docker Hub image: {image_info.full_image_name}") + + # Fetch image info + try: + repo_info = await fetch_dockerhub_info(image_info) + except HTTPException: + raise + except Exception as e: + logger.warning(f"Could not fetch Docker Hub info: {e}") + repo_info = {} + + # Fetch available tags + try: + tags = await fetch_dockerhub_tags(image_info) + except Exception as e: + logger.warning(f"Could not fetch tags: {e}") + tags = ["latest"] + + return DockerHubScanResponse( + success=True, + image_info=image_info, + description=repo_info.get('description', ''), + stars=repo_info.get('star_count', 0), + pulls=repo_info.get('pull_count', 0), + available_tags=tags, + message=f"Found Docker Hub image: {image_info.full_image_name}" + ) + + except ValueError as e: + return DockerHubScanResponse( + success=False, + error=str(e) + ) + except HTTPException: + raise + except Exception as e: + logger.error(f"Error scanning Docker Hub image: {e}") + return DockerHubScanResponse( + success=False, + error=str(e) + ) + + +@router.post("/dockerhub/register", response_model=ImportServiceResponse) +async def register_dockerhub_service( + service_name: str, + dockerhub_url: str, + tag: Optional[str] = None, + display_name: Optional[str] = None, + description: Optional[str] = None, + ports: Optional[List[Dict[str, int]]] = None, + volumes: Optional[List[Dict[str, Any]]] = None, + env_vars: Optional[List[Dict[str, Any]]] = None, + shadow_header_enabled: bool = True, + shadow_header_name: str = "X-Shadow-Service", + shadow_header_value: Optional[str] = None, + route_path: Optional[str] = None, + capabilities: Optional[List[str]] = None, + current_user: User = Depends(get_current_user) +) -> ImportServiceResponse: + """ + Register a service from Docker Hub by generating a compose file. + + Creates a docker-compose file for the specified Docker Hub image + with the provided configuration. + """ + try: + image_info = parse_dockerhub_url(dockerhub_url) + if tag: + image_info.tag = tag + + logger.info(f"Registering Docker Hub service: {service_name} from {image_info.full_image_name}") + + # Parse port configs + port_configs = [] + if ports: + for p in ports: + port_configs.append(PortConfig( + host_port=p.get('host_port', p.get('host', 8080)), + container_port=p.get('container_port', p.get('container', 8080)), + protocol=p.get('protocol', 'tcp') + )) + + # Parse volume configs + volume_configs = [] + if volumes: + for v in volumes: + volume_configs.append(VolumeConfig( + name=v.get('name', 'data'), + container_path=v.get('container_path', v.get('path', '/data')), + is_named_volume=v.get('is_named_volume', True) + )) + + # Parse env var configs + env_var_configs = [] + if env_vars: + for ev in env_vars: + env_var_configs.append(EnvVarConfigItem( + name=ev.get('name', ''), + source=ev.get('source', 'literal'), + value=ev.get('value'), + setting_path=ev.get('setting_path'), + is_secret=ev.get('is_secret', False) + )) + + # Build shadow header config + shadow_header = ShadowHeaderConfig( + enabled=shadow_header_enabled, + header_name=shadow_header_name, + header_value=shadow_header_value or service_name, + route_path=route_path or f"/{service_name}" + ) + + # Generate compose file content + compose_content = generate_compose_from_dockerhub( + image_info=image_info, + service_name=service_name, + ports=port_configs, + volumes=volume_configs, + env_vars=env_var_configs, + shadow_header=shadow_header, + display_name=display_name, + description=description, + capabilities=capabilities + ) + + # Ensure compose directory exists + compose_dir = Path('/compose') + if not compose_dir.exists(): + compose_dir = Path('compose') + compose_dir.mkdir(parents=True, exist_ok=True) + + # Generate filename + safe_name = re.sub(r'[^a-zA-Z0-9_-]', '-', service_name) + compose_filename = f"{safe_name}-compose.yaml" + compose_path = compose_dir / compose_filename + + # Write compose file + with open(compose_path, 'w') as f: + f.write(compose_content) + + # Write env file if we have literal env vars + env_file_content = [] + for ev in env_var_configs: + if ev.source == "literal" and ev.value: + env_file_content.append(f"{ev.name}={ev.value}") + + if env_file_content: + env_file_path = compose_dir / f"{safe_name}.env" + with open(env_file_path, 'w') as f: + f.write('\n'.join(env_file_content) + '\n') + + # Save service configuration to settings + from src.config.omegaconf_settings import get_settings_store + settings = get_settings_store() + + service_config_key = f"imported_services.{safe_name}" + await settings.update({ + service_config_key: { + 'source_type': 'dockerhub', + 'source_url': dockerhub_url, + 'docker_image': image_info.full_image_name, + 'compose_file': str(compose_path), + 'service_name': service_name, + 'display_name': display_name, + 'description': description, + 'shadow_header': shadow_header.model_dump(), + 'ports': [p.model_dump() for p in port_configs], + 'volumes': [v.model_dump() for v in volume_configs], + 'env_vars': [ev.model_dump() for ev in env_var_configs], + 'enabled': True + } + }) + + # Refresh compose registry to pick up new service + from src.services.compose_registry import get_compose_registry + registry = get_compose_registry() + registry.refresh() + + # Auto-install the service (mark as added so it shows in installed services) + installed_key = f"installed_services.{service_name}" + await settings.update({ + f"{installed_key}.added": True, + f"{installed_key}.enabled": True + }) + + logger.info(f"Imported and installed service '{service_name}' from Docker Hub: {image_info.full_image_name}") + + return ImportServiceResponse( + success=True, + service_id=f"{compose_filename.replace('.yaml', '')}:{service_name}", + service_name=service_name, + message=f"Successfully imported service '{service_name}' from Docker Hub", + compose_file_path=str(compose_path) + ) + + except ValueError as e: + return ImportServiceResponse( + success=False, + message=str(e) + ) + except Exception as e: + logger.error(f"Error registering Docker Hub service: {e}") + return ImportServiceResponse( + success=False, + message=f"Failed to import service: {str(e)}" + ) + + +# ============================================================================= +# Unified Endpoints (Auto-detect source type) +# ============================================================================= + +@router.post("/unified/scan", response_model=UnifiedScanResponse) +async def unified_scan( + request: UnifiedImportRequest, + current_user: User = Depends(get_current_user) +) -> UnifiedScanResponse: + """ + Scan any supported source (GitHub or Docker Hub). + + Automatically detects the source type and returns appropriate information. + """ + try: + source_type = detect_import_source(request.url) + logger.info(f"Unified scan: detected source type '{source_type}' for URL: {request.url}") + + if source_type == "github": + # Handle GitHub + github_info = parse_github_url(request.url) + if request.branch: + github_info.branch = request.branch + if request.compose_path: + github_info.path = request.compose_path + + # Fetch directory contents + contents = await fetch_github_contents(github_info.api_url) + + # Handle single file case + if isinstance(contents, dict) and contents.get('type') == 'file': + compose_files = [DetectedComposeFile( + path=contents.get('path', ''), + name=contents.get('name', ''), + download_url=contents.get('download_url', ''), + size=contents.get('size', 0) + )] + else: + compose_files = detect_compose_files(contents if isinstance(contents, list) else []) + + return UnifiedScanResponse( + success=True, + source_type="github", + github_info=github_info, + compose_files=compose_files, + message=f"Found {len(compose_files)} docker-compose file(s)" if compose_files else "No docker-compose files found" + ) + + else: + # Handle Docker Hub + image_info = parse_dockerhub_url(request.url) + if request.tag: + image_info.tag = request.tag + + # Fetch image info + try: + repo_info = await fetch_dockerhub_info(image_info) + except Exception: + repo_info = {} + + # Fetch available tags + try: + tags = await fetch_dockerhub_tags(image_info) + except Exception: + tags = ["latest"] + + return UnifiedScanResponse( + success=True, + source_type="dockerhub", + dockerhub_info=image_info, + available_tags=tags, + image_description=repo_info.get('description', ''), + message=f"Found Docker Hub image: {image_info.full_image_name}" + ) + + except ValueError as e: + return UnifiedScanResponse( + success=False, + source_type="github", + error=str(e) + ) + except HTTPException: + raise + except Exception as e: + logger.error(f"Error in unified scan: {e}") + return UnifiedScanResponse( + success=False, + source_type="github", + error=str(e) + ) diff --git a/ushadow/frontend/src/components/ImportFromGitHubModal.tsx b/ushadow/frontend/src/components/ImportFromGitHubModal.tsx new file mode 100644 index 00000000..7e0e1839 --- /dev/null +++ b/ushadow/frontend/src/components/ImportFromGitHubModal.tsx @@ -0,0 +1,1350 @@ +import { useState, useEffect } from 'react' +import { createPortal } from 'react-dom' +import { + X, + Github, + Loader2, + FileCode, + ChevronRight, + ChevronLeft, + Settings, + Key, + Check, + AlertCircle, + Server, + Eye, + EyeOff, + RefreshCw, + Box, + Plus, + Trash2, +} from 'lucide-react' +import { + githubImportApi, + DetectedComposeFile, + ComposeServiceInfo, + ShadowHeaderConfig, + EnvVarConfigItem, + DockerHubImageInfo, + PortConfig, + VolumeConfig, +} from '../services/api' + +interface ImportFromGitHubModalProps { + isOpen: boolean + onClose: () => void + onServiceImported: () => void +} + +type ImportSource = 'github' | 'dockerhub' +type WizardStep = 'url' | 'compose' | 'service' | 'config' | 'complete' + +export default function ImportFromGitHubModal({ + isOpen, + onClose, + onServiceImported, +}: ImportFromGitHubModalProps) { + // Wizard state + const [step, setStep] = useState('url') + const [loading, setLoading] = useState(false) + const [error, setError] = useState(null) + + // Source type + const [sourceType, setSourceType] = useState('github') + + // URL step + const [importUrl, setImportUrl] = useState('') + const [branch, setBranch] = useState('') + + // Docker Hub specific + const [dockerHubInfo, setDockerHubInfo] = useState(null) + const [availableTags, setAvailableTags] = useState([]) + const [selectedTag, setSelectedTag] = useState('latest') + const [imageDescription, setImageDescription] = useState('') + + // Compose file selection (GitHub) + const [composeFiles, setComposeFiles] = useState([]) + const [selectedComposeFile, setSelectedComposeFile] = useState(null) + + // Service selection (GitHub) + const [services, setServices] = useState([]) + const [selectedServices, setSelectedServices] = useState([]) + + // Configuration + const [serviceName, setServiceName] = useState('') + const [displayName, setDisplayName] = useState('') + const [description, setDescription] = useState('') + const [shadowHeader, setShadowHeader] = useState({ + enabled: true, + header_name: 'X-Shadow-Service', + header_value: '', + route_path: '', + }) + const [envVars, setEnvVars] = useState([]) + const [showSecrets, setShowSecrets] = useState>({}) + + // Docker Hub specific config + const [ports, setPorts] = useState([]) + const [volumes, setVolumes] = useState([]) + + // Capabilities this service provides + const [capabilities, setCapabilities] = useState([]) + + // Env template paste + const [showEnvPaste, setShowEnvPaste] = useState(false) + const [envPasteText, setEnvPasteText] = useState('') + + // Common capability options + const CAPABILITY_OPTIONS = [ + { id: 'llm', label: 'LLM', description: 'Language model inference' }, + { id: 'tts', label: 'TTS', description: 'Text to speech' }, + { id: 'stt', label: 'STT', description: 'Speech to text' }, + { id: 'embedding', label: 'Embedding', description: 'Text embeddings' }, + { id: 'memory', label: 'Memory', description: 'Persistent memory storage' }, + { id: 'vision', label: 'Vision', description: 'Image understanding' }, + { id: 'image_gen', label: 'Image Gen', description: 'Image generation' }, + ] + + // Reset state when modal opens/closes + useEffect(() => { + if (!isOpen) { + setStep('url') + setSourceType('github') + setImportUrl('') + setBranch('') + setDockerHubInfo(null) + setAvailableTags([]) + setSelectedTag('latest') + setImageDescription('') + setComposeFiles([]) + setSelectedComposeFile(null) + setServices([]) + setSelectedServices([]) + setServiceName('') + setDisplayName('') + setDescription('') + setShadowHeader({ + enabled: true, + header_name: 'X-Shadow-Service', + header_value: '', + route_path: '', + }) + setEnvVars([]) + setPorts([]) + setVolumes([]) + setCapabilities([]) + setShowEnvPaste(false) + setEnvPasteText('') + setError(null) + } + }, [isOpen]) + + // Helper to extract error message from various error formats + const extractErrorMessage = (err: any, fallback: string): string => { + // Handle axios error response + const detail = err?.response?.data?.detail + if (detail) { + // If detail is a string, use it directly + if (typeof detail === 'string') { + return detail + } + // If detail is an array (Pydantic validation errors), extract messages + if (Array.isArray(detail)) { + return detail + .map((e: any) => e.msg || e.message || JSON.stringify(e)) + .join(', ') + } + // If detail is an object with a message field + if (typeof detail === 'object' && detail.msg) { + return detail.msg + } + } + // Check for direct message field + if (err?.response?.data?.message) { + return err.response.data.message + } + // Check for error message on the error object itself + if (err?.message && typeof err.message === 'string') { + return err.message + } + return fallback + } + + // Auto-detect source type from URL + useEffect(() => { + const url = importUrl.toLowerCase().trim() + if (url.includes('github.com') || url.includes('githubusercontent.com')) { + setSourceType('github') + } else if (url.includes('hub.docker.com') || url.includes('docker.io') || + (url && !url.startsWith('http') && url.includes('/'))) { + setSourceType('dockerhub') + } + }, [importUrl]) + + // Scan using unified endpoint + const handleScan = async () => { + if (!importUrl.trim()) { + setError('Please enter a URL or image reference') + return + } + + setLoading(true) + setError(null) + + try { + const response = await githubImportApi.unifiedScan( + importUrl, + branch || undefined, + selectedTag !== 'latest' ? selectedTag : undefined + ) + const data = response.data + + if (!data.success) { + setError(data.error || 'Failed to scan') + return + } + + if (data.source_type === 'github') { + // GitHub flow + if (!data.compose_files || data.compose_files.length === 0) { + setError('No docker-compose files found in the repository') + return + } + setComposeFiles(data.compose_files) + setStep('compose') + } else { + // Docker Hub flow + if (data.dockerhub_info) { + setDockerHubInfo(data.dockerhub_info) + setAvailableTags(data.available_tags || ['latest']) + setImageDescription(data.image_description || '') + + // Pre-populate service name from image + const name = data.dockerhub_info.repository.replace(/[^a-zA-Z0-9]/g, '-') + setServiceName(name) + setDisplayName(name.charAt(0).toUpperCase() + name.slice(1).replace(/-/g, ' ')) + setDescription(data.image_description || `Docker Hub image: ${data.dockerhub_info.namespace}/${data.dockerhub_info.repository}`) + setShadowHeader({ + enabled: true, + header_name: 'X-Shadow-Service', + header_value: name, + route_path: `/${name}`, + }) + + // Add default port and volume + setPorts([{ host_port: 8080, container_port: 8080, protocol: 'tcp' }]) + setVolumes([{ name: 'data', container_path: '/data', is_named_volume: true }]) + + setStep('config') + } else { + setError('Could not parse Docker Hub image information') + } + } + } catch (err: any) { + setError(extractErrorMessage(err, 'Failed to scan')) + } finally { + setLoading(false) + } + } + + // Parse selected compose file (GitHub) + const handleSelectComposeFile = async (file: DetectedComposeFile) => { + setSelectedComposeFile(file) + setLoading(true) + setError(null) + + try { + const response = await githubImportApi.parse(importUrl, file.path) + const data = response.data + + if (!data.success) { + setError(data.error || 'Failed to parse compose file') + return + } + + if (data.services.length === 0) { + setError('No services found in compose file') + return + } + + setServices(data.services) + setStep('service') + } catch (err: any) { + setError(extractErrorMessage(err, 'Failed to parse compose file')) + } finally { + setLoading(false) + } + } + + // Toggle service selection (GitHub) + const handleToggleService = (service: ComposeServiceInfo) => { + setSelectedServices((prev) => { + const isSelected = prev.some((s) => s.name === service.name) + if (isSelected) { + return prev.filter((s) => s.name !== service.name) + } else { + return [...prev, service] + } + }) + } + + // Select/deselect all services + const handleSelectAllServices = () => { + if (selectedServices.length === services.length) { + setSelectedServices([]) + } else { + setSelectedServices([...services]) + } + } + + // Proceed to config after service selection + const handleProceedToConfig = () => { + if (selectedServices.length === 0) return + + // Use first selected service for basic config defaults + const firstService = selectedServices[0] + setServiceName(firstService.name) + setDisplayName(firstService.name.charAt(0).toUpperCase() + firstService.name.slice(1).replace(/-/g, ' ')) + setShadowHeader({ + enabled: true, + header_name: 'X-Shadow-Service', + header_value: firstService.name, + route_path: `/${firstService.name}`, + }) + + // Combine env vars from all selected services (deduplicated) + const allEnvVars = new Map() + for (const service of selectedServices) { + for (const env of service.environment) { + if (!allEnvVars.has(env.name)) { + allEnvVars.set(env.name, { + name: env.name, + source: env.has_default ? 'default' : 'literal', + value: env.default_value || '', + is_secret: env.name.toLowerCase().includes('key') || + env.name.toLowerCase().includes('secret') || + env.name.toLowerCase().includes('password') || + env.name.toLowerCase().includes('token'), + }) + } + } + } + setEnvVars(Array.from(allEnvVars.values())) + setStep('config') + } + + // Update environment variable + const updateEnvVar = (index: number, updates: Partial) => { + setEnvVars((prev) => + prev.map((ev, i) => (i === index ? { ...ev, ...updates } : ev)) + ) + } + + // Add new env var + const addEnvVar = () => { + setEnvVars((prev) => [...prev, { name: '', source: 'literal', value: '', is_secret: false }]) + } + + // Remove env var + const removeEnvVar = (index: number) => { + setEnvVars((prev) => prev.filter((_, i) => i !== index)) + } + + // Parse env template (KEY=value format, one per line) + const parseEnvTemplate = () => { + const lines = envPasteText.split('\n') + const newEnvVars: EnvVarConfigItem[] = [] + + for (const line of lines) { + const trimmed = line.trim() + // Skip empty lines and comments + if (!trimmed || trimmed.startsWith('#')) continue + + // Parse KEY=value or KEY= or just KEY + const match = trimmed.match(/^([A-Za-z_][A-Za-z0-9_]*)(?:=(.*))?$/) + if (match) { + const name = match[1] + const value = match[2] ?? '' + + // Check if already exists + const exists = envVars.some((e) => e.name === name) + if (!exists) { + newEnvVars.push({ + name, + source: 'literal', + value, + is_secret: name.toLowerCase().includes('key') || + name.toLowerCase().includes('secret') || + name.toLowerCase().includes('password') || + name.toLowerCase().includes('token'), + }) + } + } + } + + if (newEnvVars.length > 0) { + setEnvVars((prev) => [...prev, ...newEnvVars]) + } + setEnvPasteText('') + setShowEnvPaste(false) + } + + // Add port + const addPort = () => { + setPorts((prev) => [...prev, { host_port: 8080, container_port: 8080, protocol: 'tcp' }]) + } + + // Remove port + const removePort = (index: number) => { + setPorts((prev) => prev.filter((_, i) => i !== index)) + } + + // Update port + const updatePort = (index: number, updates: Partial) => { + setPorts((prev) => prev.map((p, i) => (i === index ? { ...p, ...updates } : p))) + } + + // Add volume + const addVolume = () => { + setVolumes((prev) => [...prev, { name: 'data', container_path: '/data', is_named_volume: true }]) + } + + // Remove volume + const removeVolume = (index: number) => { + setVolumes((prev) => prev.filter((_, i) => i !== index)) + } + + // Update volume + const updateVolume = (index: number, updates: Partial) => { + setVolumes((prev) => prev.map((v, i) => (i === index ? { ...v, ...updates } : v))) + } + + // Import the service + const handleImport = async () => { + setLoading(true) + setError(null) + + try { + if (sourceType === 'dockerhub' || dockerHubInfo) { + // Docker Hub import + const response = await githubImportApi.registerDockerHub({ + service_name: serviceName, + dockerhub_url: importUrl, + tag: selectedTag, + display_name: displayName, + description, + ports, + volumes, + env_vars: envVars, + shadow_header_enabled: shadowHeader.enabled, + shadow_header_name: shadowHeader.header_name, + shadow_header_value: shadowHeader.header_value || serviceName, + route_path: shadowHeader.route_path || `/${serviceName}`, + capabilities: capabilities.length > 0 ? capabilities : undefined, + }) + + const data = response.data + if (!data.success) { + setError(data.message || 'Failed to import service') + return + } + } else { + // GitHub import - supports multiple services + if (!selectedComposeFile || selectedServices.length === 0) { + setError('Please select a compose file and at least one service') + return + } + + const errors: string[] = [] + let successCount = 0 + + // Import each selected service + for (const service of selectedServices) { + try { + const svcName = service.name + const svcDisplayName = svcName.charAt(0).toUpperCase() + svcName.slice(1).replace(/-/g, ' ') + + const response = await githubImportApi.register({ + github_url: importUrl, + compose_path: selectedComposeFile.path, + service_name: svcName, + config: { + service_name: svcName, + display_name: svcDisplayName, + description: description || `Imported from GitHub`, + source_url: importUrl, + compose_path: selectedComposeFile.path, + shadow_header: { + ...shadowHeader, + header_value: svcName, + route_path: `/${svcName}`, + }, + env_vars: envVars, + enabled: true, + capabilities: capabilities.length > 0 ? capabilities : undefined, + }, + }) + + const data = response.data + if (data.success) { + successCount++ + } else { + errors.push(`${svcName}: ${data.message || 'Failed'}`) + } + } catch (err: any) { + errors.push(`${service.name}: ${extractErrorMessage(err, 'Failed')}`) + } + } + + // Check results + if (successCount === 0) { + setError(`Failed to import services: ${errors.join('; ')}`) + return + } + + if (errors.length > 0) { + // Partial success + setError(`Imported ${successCount} service(s), but some failed: ${errors.join('; ')}`) + } + + // Proceed to complete (even with partial success) + } + + setStep('complete') + } catch (err: any) { + setError(extractErrorMessage(err, 'Failed to import service')) + } finally { + setLoading(false) + } + } + + // Handle completion + const handleComplete = () => { + onServiceImported() + onClose() + } + + if (!isOpen) return null + + const getSteps = (): { key: WizardStep; label: string }[] => { + if (sourceType === 'dockerhub' || dockerHubInfo) { + return [ + { key: 'url', label: 'Source' }, + { key: 'config', label: 'Configure' }, + { key: 'complete', label: 'Done' }, + ] + } + return [ + { key: 'url', label: 'Source' }, + { key: 'compose', label: 'Compose' }, + { key: 'service', label: 'Service' }, + { key: 'config', label: 'Configure' }, + { key: 'complete', label: 'Done' }, + ] + } + + const renderStepIndicator = () => { + const steps = getSteps() + const currentIndex = steps.findIndex((s) => s.key === step) + + return ( +
+ {steps.map((s, index) => ( +
+
+ {index < currentIndex ? : index + 1} +
+ {index < steps.length - 1 && ( +
+ )} +
+ ))} +
+ ) + } + + const renderUrlStep = () => ( +
+ {/* Source Type Selector */} +
+ + +
+ +
+ + setImportUrl(e.target.value)} + placeholder={ + sourceType === 'github' + ? 'https://github.com/owner/repo' + : 'https://hub.docker.com/r/namespace/repo or namespace/repo' + } + className="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-lg bg-white dark:bg-gray-700 text-gray-900 dark:text-white focus:ring-2 focus:ring-primary-500 focus:border-transparent" + /> +

+ {sourceType === 'github' + ? 'Supports repository URLs, specific branches, or direct paths to compose files' + : 'Supports Docker Hub URLs or direct image references (e.g., fishaudio/fish-speech)'} +

+
+ + {sourceType === 'github' && ( +
+ + setBranch(e.target.value)} + placeholder="main" + className="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-lg bg-white dark:bg-gray-700 text-gray-900 dark:text-white focus:ring-2 focus:ring-primary-500 focus:border-transparent" + /> +
+ )} +
+ ) + + const renderComposeStep = () => ( +
+

+ Found {composeFiles.length} docker-compose file(s). Select one to continue: +

+
+ {composeFiles.map((file) => ( + + ))} +
+
+ ) + + const renderServiceStep = () => ( +
+
+

+ Found {services.length} service(s). Select the services to import: +

+ +
+
+ {services.map((service) => { + const isSelected = selectedServices.some((s) => s.name === service.name) + return ( + + ) + })} +
+ {selectedServices.length > 0 && ( +
+

+ {selectedServices.length} service(s) selected +

+ +
+ )} +
+ ) + + const renderConfigStep = () => ( +
+ {/* Docker Hub Tag Selection */} + {dockerHubInfo && availableTags.length > 0 && ( +
+

+ + Image Tag +

+ +
+ )} + + {/* Basic Info */} +
+

+ + Basic Information +

+
+
+ + setServiceName(e.target.value)} + className="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-lg bg-white dark:bg-gray-700 text-gray-900 dark:text-white text-sm" + /> +
+
+ + setDisplayName(e.target.value)} + className="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-lg bg-white dark:bg-gray-700 text-gray-900 dark:text-white text-sm" + /> +
+
+
+ +