diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 0000000..608ecf5 --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,3 @@ +# GitHub Copilot — instrucciones + +Las instrucciones de proyecto para asistentes AI están en [`CLAUDE.md`](../CLAUDE.md) en la raíz del repo. Contiene arquitectura, layout, comandos de build/test y convenciones del proyecto. diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..ecaa04f --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,3 @@ +# AGENTS.md + +Las instrucciones de proyecto para asistentes AI están en [`CLAUDE.md`](./CLAUDE.md) en la raíz del repo. Léelo antes de trabajar: contexto de arquitectura, layout, comandos y convenciones. diff --git a/CLAUDE.md b/CLAUDE.md index e2ba72a..d9692dc 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -53,12 +53,13 @@ pyinstaller bipolar-code.spec # run from repo root | Config | `core/config.py` | Reads `.env` from config dir via pydantic-settings; config dir is `C:\litellm` (Win) / `~/.litellm` (other) | | Services | `services/providers_service.py` | Provider CRUD, litellm config YAML generation, kill/start litellm subprocess | | Services | `services/proxy_service.py` | Proxy health checks; Claude Code routing (writes `ANTHROPIC_BASE_URL`/`ANTHROPIC_API_KEY` to `~/.claude/settings.json` and Windows registry) | +| Services | `services/llamacpp_service.py` | llama-server local gestionado: detección de GPUs, tensor-split auto por VRAM libre, start/stop con PID file | | API | `api/` | Thin FastAPI routers — each delegates to the matching `*_service.py` | | Models | `models/` | Pydantic schemas (`schemas.py`) and provider entity (`provider.py`) | | Startup | `main.py` | Mounts `frontend/dist/` as SPA fallback; spawns Copilot token auto-refresh background loop | | Logging | `core/logging.py` | structlog setup; use `get_logger(__name__)` throughout | -**Provider registry** persists in `{config_dir}/providers.json`. Three built-in providers: `copilot`, `anthropic`, `lmstudio`. litellm always exposes the aliases `claude-sonnet-4-6`, `claude-opus-4-6`, `gpt-4o` regardless of the active backend. +**Provider registry** persists in `{config_dir}/providers.json`. Built-in providers include `copilot`, `anthropic`, `lmstudio`, `nvidia_nim`, `openrouter`, `deepseek`, `ollama` and `llamacpp`. litellm always exposes the aliases `claude-sonnet-4-6`, `claude-opus-4-6`, `gpt-4o` regardless of the active backend. Providers with `anthropic_native: true` (llama-server, LM Studio ≥0.4.1, Ollama 2026+) receive `/v1/messages` verbatim — no litellm, no OAI translation. The `llamacpp` provider spawns a managed local `llama-server` (Vulkan multi-GPU, port 4002) via `/api/llamacpp/*`. **Platform guards**: `providers_service._start_litellm` uses PowerShell on Windows and `subprocess.Popen(start_new_session=True)` on Linux/macOS. `proxy_service._set_user_env` writes the Windows registry only on `sys.platform == "win32"`; it always writes `~/.claude/settings.json`. diff --git a/backend/app/api/llamacpp.py b/backend/app/api/llamacpp.py new file mode 100644 index 0000000..af305a3 --- /dev/null +++ b/backend/app/api/llamacpp.py @@ -0,0 +1,118 @@ +""" +Router del servidor llama.cpp local y descarga de modelos GGUF desde HF. +Thin — delega en llamacpp_service y hf_models_service. +""" +from fastapi import APIRouter, HTTPException +from pydantic import BaseModel + +from app.core.logging import get_logger +from app.core.utils import sanitize_error +from app.services import hf_models_service, llamacpp_service, providers_service + +log = get_logger(__name__) +router = APIRouter(prefix="/llamacpp", tags=["llamacpp"]) + + +class DownloadRequest(BaseModel): + repo_id: str + filename: str + + +class UseModelRequest(BaseModel): + path: str + + +def _llamacpp_provider(): + provider = providers_service.get_provider("llamacpp") + if not provider: + raise HTTPException(status_code=404, detail="Provider 'llamacpp' no registrado") + return provider + + +@router.get("/devices") +async def devices(): + provider = _llamacpp_provider() + exe = llamacpp_service.resolve_exe(provider) + return { + "exe_found": bool(exe), + "exe_path": exe, + "devices": llamacpp_service.list_devices(exe), + } + + +@router.get("/status") +async def status(): + return await llamacpp_service.get_status(_llamacpp_provider()) + + +@router.post("/start") +async def start(): + try: + return await llamacpp_service.start_llamacpp(_llamacpp_provider()) + except ValueError as e: + raise HTTPException(status_code=400, detail=sanitize_error(str(e))) + except Exception as e: + log.error("llamacpp_start_failed", error=str(e)) + raise HTTPException(status_code=500, detail=sanitize_error(str(e))) + + +@router.post("/stop") +async def stop(force: bool = False): + try: + return await llamacpp_service.stop_llamacpp(_llamacpp_provider(), force=force) + except RuntimeError as e: + raise HTTPException(status_code=409, detail=sanitize_error(str(e))) + + +@router.get("/logs") +async def logs(lines: int = 80): + return {"logs": llamacpp_service.tail_logs(lines)} + + +# ── Modelos GGUF (Hugging Face) ────────────────────────────────────────────── + +@router.get("/hf/search") +async def hf_search(q: str): + try: + return {"results": await hf_models_service.search_gguf(q)} + except Exception as e: + raise HTTPException(status_code=502, detail=sanitize_error(str(e))) + + +@router.get("/hf/files") +async def hf_files(repo_id: str): + try: + return {"files": await hf_models_service.list_repo_gguf_files(repo_id)} + except Exception as e: + raise HTTPException(status_code=502, detail=sanitize_error(str(e))) + + +@router.post("/hf/download") +async def hf_download(body: DownloadRequest): + try: + return await hf_models_service.start_download(body.repo_id, body.filename) + except Exception as e: + raise HTTPException(status_code=502, detail=sanitize_error(str(e))) + + +@router.get("/hf/downloads") +async def hf_downloads(): + return {"downloads": hf_models_service.get_downloads()} + + +@router.delete("/hf/download/{download_id:path}") +async def hf_cancel(download_id: str): + return {"cancelled": hf_models_service.cancel_download(download_id)} + + +@router.get("/models") +async def local_models(): + return {"models": hf_models_service.list_local_models()} + + +@router.post("/use-model") +async def use_model(body: UseModelRequest): + provider = _llamacpp_provider() + launch = {**provider.local_launch, "model_path": body.path} + updated = providers_service.update_provider(provider.id, {"local_launch": launch}) + return {"model_path": updated.local_launch.get("model_path", "")} diff --git a/backend/app/api/messages.py b/backend/app/api/messages.py index 7ff3455..3c74d95 100644 --- a/backend/app/api/messages.py +++ b/backend/app/api/messages.py @@ -326,6 +326,7 @@ async def messages_passthrough(request: Request): active = providers_service.get_active_provider() active_provider_id = active.id if active else "unknown" is_anthropic = active and active.litellm_prefix == "anthropic" + is_native = bool(active and active.anthropic_native) ctx_window = token_service.get_context_window(model) used = token_service.count_tokens(messages) @@ -351,18 +352,30 @@ async def generate(): timeout = httpx.Timeout(connect=10.0, read=120.0, write=10.0, pool=10.0) async with httpx.AsyncClient(timeout=timeout) as client: - if is_anthropic: - # Anthropic provider: passthrough to litellm /v1/messages - forward_headers = { - "Authorization": f"Bearer {settings.proxy_api_key}", - "Content-Type": "application/json", - } + if is_anthropic or is_native: + if is_native: + # Provider con /v1/messages nativo (llama-server, LM Studio >=0.4.1, + # Ollama 2026+): reenvío verbatim, solo se reescribe el model + body["model"] = active.active_model or model + native_base = active.api_base.rstrip("/").removesuffix("/v1") + target_url = f"{native_base}/v1/messages" + forward_headers = {"Content-Type": "application/json"} + native_key = os.environ.get(active.auth_env_var, "") if active.auth_env_var else "" + if native_key: + forward_headers["x-api-key"] = native_key + else: + # Anthropic provider: passthrough to litellm /v1/messages + target_url = f"{settings.proxy_url}/v1/messages" + forward_headers = { + "Authorization": f"Bearer {settings.proxy_api_key}", + "Content-Type": "application/json", + } for h in ("anthropic-version", "anthropic-beta"): if h in request.headers: forward_headers[h] = request.headers[h] async with client.stream( - "POST", f"{settings.proxy_url}/v1/messages", + "POST", target_url, json=body, headers=forward_headers, ) as resp: if resp.status_code >= 400: @@ -505,6 +518,16 @@ async def generate(): _record_usage(active_provider_id, model, usage_buf, truncated) + except httpx.ConnectError as e: + if is_native and active: + log.error("native_provider_unreachable", api_base=active.api_base) + yield _sse_error( + f"Servidor local no responde en {active.api_base}. " + "Inícialo desde Providers → llama.cpp (Start)." + ) + else: + log.error("messages_passthrough_error", error=_sanitize_error(str(e))) + yield _sse_error(_sanitize_error(str(e))) except Exception as e: log.error("messages_passthrough_error", error=_sanitize_error(str(e))) yield _sse_error(_sanitize_error(str(e))) diff --git a/backend/app/api/settings.py b/backend/app/api/settings.py index a23a8a5..2a56f86 100644 --- a/backend/app/api/settings.py +++ b/backend/app/api/settings.py @@ -46,3 +46,22 @@ async def get_full_api_key(): """Devuelve el API key completo para configurar Claude Code en PCs remotos.""" s = _get_settings() return {"api_key": s.ui_api_key} + + +@router.get("/connection-info") +async def get_connection_info(): + """IP LAN + puerto para configurar Claude Code desde otros PCs.""" + import socket + lan_ip = "" + try: + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock: + # No envía tráfico: connect en UDP solo resuelve la interfaz de salida + sock.connect(("8.8.8.8", 80)) + lan_ip = sock.getsockname()[0] + except OSError as e: + log.warning("lan_ip_detection_failed", error=str(e)) + return { + "lan_ip": lan_ip, + "port": 8000, + "anthropic_base_url": f"http://{lan_ip}:8000" if lan_ip else "", + } diff --git a/backend/app/main.py b/backend/app/main.py index 947876e..b8cf842 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -11,6 +11,7 @@ from app.api import providers as providers_router from app.api import messages as messages_router from app.api import pricing as pricing_router +from app.api import llamacpp as llamacpp_router from app.core.logging import setup_logging, get_logger from app.core.config import get_settings @@ -62,7 +63,9 @@ async def _copilot_token_refresh_loop(): @asynccontextmanager async def lifespan(app: FastAPI): from app.services import usage_tracker + from app.services.llamacpp_service import autostart_if_configured await usage_tracker.init_db() + asyncio.create_task(autostart_if_configured()) task = asyncio.create_task(_copilot_token_refresh_loop()) try: yield @@ -120,6 +123,7 @@ def create_app() -> FastAPI: app.include_router(chat_router.router, prefix="/api") app.include_router(messages_router.router) app.include_router(pricing_router.router, prefix="/api") + app.include_router(llamacpp_router.router, prefix="/api") @app.get("/api/health") async def health(): diff --git a/backend/app/models/provider.py b/backend/app/models/provider.py index 35ebd56..db8ac1f 100644 --- a/backend/app/models/provider.py +++ b/backend/app/models/provider.py @@ -33,6 +33,14 @@ class Provider(BaseModel): max_tools: int = 0 # 0 = sin límite; >0 trunca el array de tools al enviarlo rate_limit_rpm: int = 0 # límite externo del proveedor en req/min (0 = desconocido) + # Providers con Anthropic Messages API nativa (/v1/messages): llama-server, + # LM Studio >=0.4.1, Ollama 2026+. El body se reenvía verbatim sin traducción. + anthropic_native: bool = False + + # Config de lanzamiento para providers locales gestionados (llama.cpp): + # {exe_path, model_path, ctx_size, split_mode, tensor_split, ngl, extra_args} + local_launch: dict = {} + class ProviderRegistry(BaseModel): active_provider_id: str = "copilot" diff --git a/backend/app/services/hf_models_service.py b/backend/app/services/hf_models_service.py new file mode 100644 index 0000000..d603c5a --- /dev/null +++ b/backend/app/services/hf_models_service.py @@ -0,0 +1,206 @@ +""" +Búsqueda y descarga de modelos GGUF desde Hugging Face al dir local de modelos. +Descargas en background con progreso en memoria y resume por Range sobre .part. +""" +import asyncio +import os +import re +import time +from pathlib import Path + +import httpx + +from app.core.config import get_settings +from app.core.logging import get_logger + +log = get_logger(__name__) + +HF_API = "https://huggingface.co/api/models" +_MULTIPART = re.compile(r"-(\d{5})-of-(\d{5})\.gguf$") +_CHUNK_SIZE = 1024 * 1024 + +_downloads: dict[str, dict] = {} +_tasks: dict[str, asyncio.Task] = {} + + +def models_dir() -> Path: + path = Path(get_settings().litellm_config_dir) / "models" + path.mkdir(parents=True, exist_ok=True) + return path + + +def _hf_headers() -> dict: + token = os.environ.get("HF_TOKEN", "") + return {"Authorization": f"Bearer {token}"} if token else {} + + +async def search_gguf(query: str, limit: int = 20) -> list[dict]: + params = { + "search": query, + "filter": "gguf", + "sort": "downloads", + "direction": "-1", + "limit": str(limit), + } + async with httpx.AsyncClient(timeout=15.0) as client: + resp = await client.get(HF_API, params=params, headers=_hf_headers()) + resp.raise_for_status() + return [ + { + "id": m.get("id", ""), + "downloads": m.get("downloads", 0), + "likes": m.get("likes", 0), + "updated": m.get("lastModified", ""), + } + for m in resp.json() + ] + + +async def list_repo_gguf_files(repo_id: str) -> list[dict]: + url = f"{HF_API}/{repo_id}" + async with httpx.AsyncClient(timeout=15.0) as client: + resp = await client.get(url, params={"blobs": "true"}, headers=_hf_headers()) + resp.raise_for_status() + siblings = resp.json().get("siblings", []) + files = [ + {"filename": s["rfilename"], "size": s.get("size") or 0} + for s in siblings + if s.get("rfilename", "").endswith(".gguf") + ] + return group_multipart(files) + + +def group_multipart(files: list[dict]) -> list[dict]: + """Agrupa GGUFs multi-parte (-00001-of-0000N) en una sola entrada descargable.""" + groups: dict[str, dict] = {} + singles: list[dict] = [] + for f in files: + m = _MULTIPART.search(f["filename"]) + if not m: + singles.append({**f, "parts": [f["filename"]]}) + continue + base = _MULTIPART.sub("", f["filename"]) + group = groups.setdefault(base, {"filename": base, "size": 0, "parts": []}) + group["size"] += f["size"] + group["parts"].append(f["filename"]) + for group in groups.values(): + group["parts"].sort() + return sorted(singles + list(groups.values()), key=lambda x: x["filename"]) + + +def expand_parts(repo_id: str, filename: str, files: list[dict]) -> list[str]: + for f in files: + if f["filename"] == filename: + return f["parts"] + return [filename] + + +def list_local_models() -> list[dict]: + result = [] + for path in sorted(models_dir().glob("*.gguf")): + m = _MULTIPART.search(path.name) + if m and m.group(1) != "00001": + continue # solo la primera parte es cargable por llama.cpp + result.append({"filename": path.name, "path": str(path), "size": path.stat().st_size}) + return result + + +def download_id(repo_id: str, filename: str) -> str: + return f"{repo_id}::{filename}" + + +def get_downloads() -> list[dict]: + return list(_downloads.values()) + + +async def start_download(repo_id: str, filename: str) -> dict: + did = download_id(repo_id, filename) + existing = _downloads.get(did) + if existing and existing["status"] in ("downloading", "queued"): + return existing + + files = await list_repo_gguf_files(repo_id) + parts = expand_parts(repo_id, filename, files) + total = next((f["size"] for f in files if f["filename"] == filename), 0) + + state = { + "id": did, + "repo_id": repo_id, + "filename": filename, + "parts": parts, + "status": "queued", + "total_bytes": total, + "downloaded_bytes": 0, + "speed_bps": 0, + "error": "", + } + _downloads[did] = state + task = asyncio.create_task(_run_download(state)) + _tasks[did] = task + task.add_done_callback(lambda _: _tasks.pop(did, None)) + return state + + +def cancel_download(did: str) -> bool: + task = _tasks.get(did) + state = _downloads.get(did) + if task and not task.done(): + task.cancel() + if state: + state["status"] = "cancelled" + return True + return False + + +async def _run_download(state: dict) -> None: + state["status"] = "downloading" + try: + for part in state["parts"]: + await _download_file(state, part) + state["status"] = "done" + log.info("hf_download_done", id=state["id"], bytes=state["downloaded_bytes"]) + except asyncio.CancelledError: + state["status"] = "cancelled" + raise + except Exception as e: + state["status"] = "error" + state["error"] = str(e)[:300] + log.error("hf_download_failed", id=state["id"], error=str(e)) + + +async def _download_file(state: dict, part: str) -> None: + url = f"https://huggingface.co/{state['repo_id']}/resolve/main/{part}?download=true" + dest = models_dir() / Path(part).name + tmp = dest.with_suffix(dest.suffix + ".part") + if dest.exists(): + state["downloaded_bytes"] += dest.stat().st_size + return + + resume_from = tmp.stat().st_size if tmp.exists() else 0 + headers = _hf_headers() + if resume_from: + headers["Range"] = f"bytes={resume_from}-" + state["downloaded_bytes"] += resume_from + + last_tick = time.monotonic() + tick_bytes = 0 + timeout = httpx.Timeout(connect=15.0, read=60.0, write=15.0, pool=15.0) + async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client: + async with client.stream("GET", url, headers=headers) as resp: + if resp.status_code == 416: + # .part ya completo: renombrar y seguir + tmp.rename(dest) + return + resp.raise_for_status() + mode = "ab" if resume_from else "wb" + with open(tmp, mode) as f: + async for chunk in resp.aiter_bytes(_CHUNK_SIZE): + f.write(chunk) + state["downloaded_bytes"] += len(chunk) + tick_bytes += len(chunk) + now = time.monotonic() + if now - last_tick >= 1.0: + state["speed_bps"] = int(tick_bytes / (now - last_tick)) + last_tick = now + tick_bytes = 0 + tmp.rename(dest) diff --git a/backend/app/services/llamacpp_service.py b/backend/app/services/llamacpp_service.py new file mode 100644 index 0000000..90fce77 --- /dev/null +++ b/backend/app/services/llamacpp_service.py @@ -0,0 +1,283 @@ +""" +Gestión del servidor llama.cpp local (llama-server): detección de GPUs, +tensor-split proporcional a VRAM libre, start/stop/status con PID file. +Espejo del patrón usado para litellm en providers_service. +""" +import re +import shutil +import subprocess +import sys +from pathlib import Path + +import psutil + +from app.core.config import get_settings +from app.core.logging import get_logger +from app.models.provider import Provider + +log = get_logger(__name__) + +DEFAULT_PORT = 4002 +# Heurística KV cache: ~40 KiB/token cubre modelos GQA 30-80B en q8 KV. +_KV_MIB_PER_TOKEN = 0.04 +_MODEL_OVERHEAD_FACTOR = 1.15 + +_DEVICE_LINE = re.compile( + r"^\s*([A-Za-z]+)(\d+):\s+(.+?)\s+\((\d+)\s+MiB,\s+(\d+)\s+MiB free\)" +) + + +def _config_dir() -> Path: + return Path(get_settings().litellm_config_dir) + + +def _pid_file() -> Path: + return _config_dir() / "llamacpp.pid" + + +def resolve_exe(provider: Provider) -> str: + configured = str(provider.local_launch.get("exe_path", "")).strip() + if configured: + return configured + found = shutil.which("llama-server") + return found or "" + + +def port_from_api_base(api_base: str) -> int: + match = re.search(r":(\d+)", api_base.split("//")[-1]) + return int(match.group(1)) if match else DEFAULT_PORT + + +def parse_devices(output: str) -> list[dict]: + devices = [] + for line in output.splitlines(): + m = _DEVICE_LINE.match(line) + if not m: + continue + devices.append({ + "index": int(m.group(2)), + "backend": m.group(1), + "name": m.group(3).strip(), + "vram_total_mib": int(m.group(4)), + "vram_free_mib": int(m.group(5)), + }) + return devices + + +def list_devices(exe: str) -> list[dict]: + if not exe: + return [] + try: + result = subprocess.run( + [exe, "--list-devices"], + capture_output=True, text=True, timeout=30, + creationflags=subprocess.CREATE_NO_WINDOW if sys.platform == "win32" else 0, + ) + return parse_devices(result.stdout + "\n" + result.stderr) + except (OSError, subprocess.TimeoutExpired) as e: + log.warning("list_devices_failed", exe=exe, error=str(e)) + return [] + + +def compute_tensor_split(devices: list[dict]) -> list[float]: + frees = [max(0, d["vram_free_mib"]) for d in devices] + total = sum(frees) + if total <= 0: + return [] + return [round(f / total, 3) for f in frees] + + +def estimate_fit(model_path: str, ctx_size: int, devices: list[dict]) -> dict: + path = Path(model_path) + model_mib = path.stat().st_size / (1024 * 1024) if path.exists() else 0 + needed = int(model_mib * _MODEL_OVERHEAD_FACTOR + ctx_size * _KV_MIB_PER_TOKEN) + available = sum(d["vram_free_mib"] for d in devices) + return { + "fits": needed <= available if model_mib > 0 else False, + "needed_mib": needed, + "available_mib": available, + } + + +def build_cmdline(provider: Provider, devices: list[dict]) -> list[str]: + launch = provider.local_launch + exe = resolve_exe(provider) + port = port_from_api_base(provider.api_base) + ctx_size = int(launch.get("ctx_size", 32768)) + ngl = int(launch.get("ngl", 999)) + split_mode = str(launch.get("split_mode", "layer")) + + cmd = [ + exe, + "--model", str(launch.get("model_path", "")), + "--ctx-size", str(ctx_size), + "--n-gpu-layers", str(ngl), + "--host", str(launch.get("host", "127.0.0.1")), + "--port", str(port), + "--jinja", + "--slots", + ] + + if len(devices) > 1: + cmd += ["--split-mode", split_mode] + split = launch.get("tensor_split", "auto") + ratios = compute_tensor_split(devices) if split == "auto" else [float(x) for x in split] + if ratios: + cmd += ["--tensor-split", ",".join(str(r) for r in ratios)] + + extra = launch.get("extra_args", []) + if isinstance(extra, list): + cmd += [str(a) for a in extra] + return cmd + + +async def get_status(provider: Provider) -> dict: + import httpx + + pid = _read_pid() + running = pid is not None and psutil.pid_exists(pid) + port = port_from_api_base(provider.api_base) + status = { + "running": running, + "pid": pid if running else None, + "port": port, + "model_path": provider.local_launch.get("model_path") or None, + "healthy": False, + "busy_slots": -1, + } + if not running: + return status + + base = provider.api_base.rstrip("/") + if base.endswith("/v1"): + base = base[:-3] + async with httpx.AsyncClient(timeout=httpx.Timeout(3.0)) as client: + try: + resp = await client.get(f"{base}/health") + status["healthy"] = resp.status_code == 200 + except httpx.HTTPError: + return status + try: + resp = await client.get(f"{base}/slots") + if resp.status_code == 200: + slots = resp.json() + status["busy_slots"] = sum(1 for s in slots if s.get("is_processing")) + except (httpx.HTTPError, ValueError): + pass + return status + + +async def start_llamacpp(provider: Provider) -> dict: + exe = resolve_exe(provider) + if not exe: + raise ValueError( + "llama-server no encontrado. Instala un release Vulkan de llama.cpp " + "y configura exe_path en el provider." + ) + model_path = str(provider.local_launch.get("model_path", "")).strip() + if not model_path or not Path(model_path).exists(): + raise ValueError(f"Modelo GGUF no encontrado: '{model_path}'") + + current = await get_status(provider) + if current["running"]: + return {**current, "already_running": True} + + devices = list_devices(exe) + fit = estimate_fit(model_path, int(provider.local_launch.get("ctx_size", 32768)), devices) + if devices and not fit["fits"]: + log.warning("model_may_not_fit", **fit) + + cmd = build_cmdline(provider, devices) + out_log = _config_dir() / "llamacpp-out.log" + err_log = _config_dir() / "llamacpp-err.log" + _spawn_detached(cmd, out_log, err_log) + log.info("llamacpp_started", cmd=" ".join(cmd), devices=len(devices), fit=fit) + return {"started": True, "cmdline": cmd, "devices": devices, "fit": fit} + + +def _spawn_detached(cmd: list[str], out_log: Path, err_log: Path) -> None: + with open(out_log, "ab") as fout, open(err_log, "ab") as ferr: + if sys.platform == "win32": + flags = subprocess.CREATE_NO_WINDOW | subprocess.CREATE_NEW_PROCESS_GROUP + proc = subprocess.Popen( + cmd, stdout=fout, stderr=ferr, stdin=subprocess.DEVNULL, + creationflags=flags, + ) + else: + proc = subprocess.Popen( + cmd, stdout=fout, stderr=ferr, stdin=subprocess.DEVNULL, + start_new_session=True, + ) + _pid_file().write_text(str(proc.pid), encoding="utf-8") + + +async def stop_llamacpp(provider: Provider, force: bool = False) -> dict: + status = await get_status(provider) + if status["running"] and status["busy_slots"] > 0 and not force: + raise RuntimeError( + f"llama-server tiene {status['busy_slots']} slot(s) procesando. " + "Usa force=true para detenerlo de todos modos." + ) + + killed = 0 + pid = _read_pid() + if pid is not None: + try: + psutil.Process(pid).kill() + killed += 1 + except (psutil.NoSuchProcess, psutil.AccessDenied) as e: + log.warning("llamacpp_pid_kill_failed", pid=pid, error=str(e)) + _pid_file().unlink(missing_ok=True) + + for proc in psutil.process_iter(["pid", "name", "cmdline"]): + try: + name = (proc.info.get("name") or "").lower() + if name.startswith("llama-server"): + proc.kill() + killed += 1 + except (psutil.NoSuchProcess, psutil.AccessDenied): + pass + + log.info("llamacpp_stopped", killed=killed, forced=force) + return {"stopped": True, "killed": killed} + + +def _read_pid() -> int | None: + try: + return int(_pid_file().read_text().strip()) + except (FileNotFoundError, ValueError): + return None + + +def _tail_file(path: Path, lines: int) -> list[str]: + try: + with open(path, "rb") as f: + f.seek(0, 2) + size = f.tell() + f.seek(max(0, size - 65536)) + data = f.read().decode("utf-8", errors="replace") + return data.splitlines()[-lines:] + except OSError: + return [] + + +def tail_logs(lines: int = 80) -> list[str]: + merged = [] + for name, tag in (("llamacpp-out.log", "out"), ("llamacpp-err.log", "err")): + merged += [f"[{tag}] {line}" for line in _tail_file(_config_dir() / name, lines)] + return merged[-lines:] + + +async def autostart_if_configured() -> None: + """Arranque al boot del backend: solo si el provider llamacpp lo pide explícitamente.""" + from app.services import providers_service + provider = providers_service.get_provider("llamacpp") + if not provider or not provider.local_launch.get("autostart"): + return + if not str(provider.local_launch.get("model_path", "")).strip(): + return + try: + await start_llamacpp(provider) + log.info("llamacpp_autostarted") + except Exception as e: + log.warning("llamacpp_autostart_failed", error=str(e)) diff --git a/backend/app/services/providers_service.py b/backend/app/services/providers_service.py index 3fe6bf9..7c36cb9 100644 --- a/backend/app/services/providers_service.py +++ b/backend/app/services/providers_service.py @@ -96,6 +96,27 @@ "active_model": "deepseek-chat", "drop_params": True, }, + { + "id": "llamacpp", + "name": "llama.cpp (Local multi-GPU)", + "description": "llama-server gestionado — Vulkan multi-GPU (tensor-split), Anthropic Messages API nativa", + "api_base": "http://127.0.0.1:4002", + "litellm_prefix": "openai", + "auth_env_var": "", + "models_endpoint": "http://127.0.0.1:4002/v1/models", + "active_model": "local", + "anthropic_native": True, + "drop_params": True, + "local_launch": { + "exe_path": "", + "model_path": "", + "ctx_size": 32768, + "split_mode": "layer", + "tensor_split": "auto", + "ngl": 999, + "extra_args": [], + }, + }, { "id": "ollama", "name": "Ollama (Local)", diff --git a/backend/tests/test_hf_models_service.py b/backend/tests/test_hf_models_service.py new file mode 100644 index 0000000..67bd9fb --- /dev/null +++ b/backend/tests/test_hf_models_service.py @@ -0,0 +1,163 @@ +from types import SimpleNamespace +from unittest.mock import AsyncMock, patch + +import pytest + +from app.services import hf_models_service + + +@pytest.fixture(autouse=True) +def reset_download_state(): + hf_models_service._downloads.clear() + hf_models_service._tasks.clear() + yield + hf_models_service._downloads.clear() + hf_models_service._tasks.clear() + + +def test_group_multipart_single_file_passes_through_with_itself_as_part(): + files = [{"filename": "single.gguf", "size": 42}] + + grouped = hf_models_service.group_multipart(files) + + assert grouped == [ + { + "filename": "single.gguf", + "size": 42, + "parts": ["single.gguf"], + } + ] + + +def test_group_multipart_collapses_parts_using_regex_base(): + files = [ + {"filename": "m-00002-of-00003.gguf", "size": 20}, + {"filename": "m-00003-of-00003.gguf", "size": 30}, + {"filename": "m-00001-of-00003.gguf", "size": 10}, + ] + + grouped = hf_models_service.group_multipart(files) + + assert grouped == [ + { + "filename": "m", + "size": 60, + "parts": [ + "m-00001-of-00003.gguf", + "m-00002-of-00003.gguf", + "m-00003-of-00003.gguf", + ], + } + ] + + +def test_group_multipart_mixed_files_are_sorted_by_filename(): + files = [ + {"filename": "z.gguf", "size": 5}, + {"filename": "m-00002-of-00002.gguf", "size": 20}, + {"filename": "a.gguf", "size": 1}, + {"filename": "m-00001-of-00002.gguf", "size": 10}, + ] + + grouped = hf_models_service.group_multipart(files) + + assert [file["filename"] for file in grouped] == ["a.gguf", "m", "z.gguf"] + + +def test_expand_parts_returns_parts_for_grouped_filename(): + parts = [ + "m-00001-of-00002.gguf", + "m-00002-of-00002.gguf", + ] + files = [{"filename": "m", "size": 30, "parts": parts}] + + assert hf_models_service.expand_parts("owner/repo", "m", files) == parts + + +def test_expand_parts_returns_filename_when_group_is_not_found(): + files = [{"filename": "other.gguf", "size": 10, "parts": ["other.gguf"]}] + + assert hf_models_service.expand_parts("owner/repo", "missing.gguf", files) == [ + "missing.gguf" + ] + + +def test_download_id_joins_repo_and_filename_with_double_colon(): + assert ( + hf_models_service.download_id("owner/repo", "model.gguf") + == "owner/repo::model.gguf" + ) + + +def test_list_local_models_includes_single_and_first_multipart_file( + tmp_path, + monkeypatch, +): + model_dir = tmp_path / "models" + model_dir.mkdir() + (model_dir / "a.gguf").write_bytes(b"single") + (model_dir / "m-00001-of-00002.gguf").write_bytes(b"first") + (model_dir / "m-00002-of-00002.gguf").write_bytes(b"second") + monkeypatch.setattr( + hf_models_service, + "get_settings", + lambda: SimpleNamespace(litellm_config_dir=tmp_path), + ) + + models = hf_models_service.list_local_models() + + assert [model["filename"] for model in models] == [ + "a.gguf", + "m-00001-of-00002.gguf", + ] + assert "m-00002-of-00002.gguf" not in { + model["filename"] for model in models + } + + +def test_cancel_download_returns_false_without_matching_task(): + assert hf_models_service.cancel_download("owner/repo::model.gguf") is False + + +def test_hf_headers_uses_bearer_token(monkeypatch): + monkeypatch.setenv("HF_TOKEN", "secret-token") + + assert hf_models_service._hf_headers() == { + "Authorization": "Bearer secret-token" + } + + +def test_hf_headers_without_token_returns_empty_dict(monkeypatch): + monkeypatch.delenv("HF_TOKEN", raising=False) + + assert hf_models_service._hf_headers() == {} + + +@pytest.mark.asyncio +async def test_start_download_returns_existing_active_download_without_new_task(): + did = hf_models_service.download_id("owner/repo", "model.gguf") + existing = { + "id": did, + "repo_id": "owner/repo", + "filename": "model.gguf", + "status": "downloading", + } + hf_models_service._downloads[did] = existing + + with ( + patch.object( + hf_models_service, + "list_repo_gguf_files", + AsyncMock(), + ) as list_repo_files_mock, + patch.object(hf_models_service.asyncio, "create_task") as create_task_mock, + ): + result = await hf_models_service.start_download( + "owner/repo", + "model.gguf", + ) + + assert result is existing + list_repo_files_mock.assert_not_awaited() + create_task_mock.assert_not_called() + assert hf_models_service._tasks == {} diff --git a/backend/tests/test_llamacpp_service.py b/backend/tests/test_llamacpp_service.py new file mode 100644 index 0000000..e56aab4 --- /dev/null +++ b/backend/tests/test_llamacpp_service.py @@ -0,0 +1,248 @@ +from unittest.mock import AsyncMock, patch + +import pytest + +from app.models.provider import Provider +from app.services import llamacpp_service + + +DEVICE_OUTPUT = ( + "Available devices:\n" + " Vulkan0: AMD Radeon AI PRO R9700 (32768 MiB, 31900 MiB free)\n" + " Vulkan1: AMD Radeon RX 7800 XT (16384 MiB, 16100 MiB free)\n" +) + + +@pytest.fixture +def provider_factory(): + def create(local_launch=None, api_base="http://127.0.0.1:4002"): + return Provider( + id="llamacpp", + name="Local llama.cpp", + api_base=api_base, + local_launch=local_launch or {}, + ) + + return create + + +def test_parse_devices_returns_expected_device_fields(): + devices = llamacpp_service.parse_devices(DEVICE_OUTPUT) + + assert devices == [ + { + "index": 0, + "backend": "Vulkan", + "name": "AMD Radeon AI PRO R9700", + "vram_total_mib": 32768, + "vram_free_mib": 31900, + }, + { + "index": 1, + "backend": "Vulkan", + "name": "AMD Radeon RX 7800 XT", + "vram_total_mib": 16384, + "vram_free_mib": 16100, + }, + ] + + +def test_parse_devices_ignores_garbage_lines(): + output = "garbage\n" + DEVICE_OUTPUT + "VulkanX: malformed device\n" + + devices = llamacpp_service.parse_devices(output) + + assert [device["index"] for device in devices] == [0, 1] + + +def test_parse_devices_empty_output_returns_empty_list(): + assert llamacpp_service.parse_devices("") == [] + + +def test_compute_tensor_split_is_proportional_and_normalized(): + devices = [ + {"vram_free_mib": 31900}, + {"vram_free_mib": 16100}, + ] + + split = llamacpp_service.compute_tensor_split(devices) + + assert split == pytest.approx([0.665, 0.335], abs=0.001) + assert sum(split) == pytest.approx(1.0) + + +def test_compute_tensor_split_empty_devices_returns_empty_list(): + assert llamacpp_service.compute_tensor_split([]) == [] + + +def test_compute_tensor_split_all_zero_free_vram_returns_empty_list(): + devices = [ + {"vram_free_mib": 0}, + {"vram_free_mib": 0}, + ] + + assert llamacpp_service.compute_tensor_split(devices) == [] + + +def test_port_from_api_base_uses_explicit_port(): + assert llamacpp_service.port_from_api_base("http://127.0.0.1:4002") == 4002 + + +def test_port_from_api_base_without_port_uses_default(): + assert ( + llamacpp_service.port_from_api_base("http://x/v1") + == llamacpp_service.DEFAULT_PORT + ) + + +def test_estimate_fit_accounts_for_model_and_context_size(tmp_path): + model_path = tmp_path / "model.gguf" + with model_path.open("wb") as model_file: + model_file.truncate(20 * 1024 * 1024) + devices = [{"vram_free_mib": 70}] + + without_context = llamacpp_service.estimate_fit(str(model_path), 0, devices) + with_context = llamacpp_service.estimate_fit(str(model_path), 1000, devices) + + assert without_context["needed_mib"] == 23 + assert with_context == { + "fits": True, + "needed_mib": 63, + "available_mib": 70, + } + assert with_context["needed_mib"] - without_context["needed_mib"] == 40 + + +def test_estimate_fit_missing_model_never_fits(tmp_path): + missing_model = tmp_path / "missing.gguf" + + estimate = llamacpp_service.estimate_fit( + str(missing_model), + 100, + [{"vram_free_mib": 999}], + ) + + assert estimate == { + "fits": False, + "needed_mib": 4, + "available_mib": 999, + } + + +def test_build_cmdline_single_device_uses_defaults_without_tensor_split( + provider_factory, +): + provider = provider_factory( + local_launch={ + "exe_path": "llama-server", + "model_path": "model.gguf", + } + ) + + cmdline = llamacpp_service.build_cmdline( + provider, + [{"vram_free_mib": 31900}], + ) + + assert "--tensor-split" not in cmdline + assert "--split-mode" not in cmdline + assert cmdline[cmdline.index("--ctx-size") + 1] == "32768" + assert cmdline[cmdline.index("--n-gpu-layers") + 1] == "999" + assert "--jinja" in cmdline + assert "--slots" in cmdline + + +def test_build_cmdline_two_devices_uses_auto_tensor_split(provider_factory): + provider = provider_factory( + local_launch={ + "exe_path": "llama-server", + "model_path": "model.gguf", + "tensor_split": "auto", + } + ) + devices = [ + {"vram_free_mib": 31900}, + {"vram_free_mib": 16100}, + ] + + cmdline = llamacpp_service.build_cmdline(provider, devices) + + assert cmdline[cmdline.index("--split-mode") + 1] == "layer" + assert cmdline[cmdline.index("--tensor-split") + 1] == "0.665,0.335" + assert "--jinja" in cmdline + assert "--slots" in cmdline + + +def test_build_cmdline_respects_manual_tensor_split(provider_factory): + provider = provider_factory( + local_launch={ + "exe_path": "llama-server", + "model_path": "model.gguf", + "tensor_split": [2, 1], + } + ) + devices = [ + {"vram_free_mib": 1}, + {"vram_free_mib": 1}, + ] + + cmdline = llamacpp_service.build_cmdline(provider, devices) + + assert cmdline[cmdline.index("--tensor-split") + 1] == "2.0,1.0" + assert "--jinja" in cmdline + assert "--slots" in cmdline + + +@pytest.mark.asyncio +async def test_stop_llamacpp_refuses_busy_server_without_force(provider_factory): + provider = provider_factory() + status = {"running": True, "busy_slots": 2} + + with ( + patch.object( + llamacpp_service, + "get_status", + AsyncMock(return_value=status), + ), + patch.object(llamacpp_service.psutil, "Process") as process_mock, + patch.object(llamacpp_service.psutil, "process_iter") as process_iter_mock, + ): + with pytest.raises(RuntimeError, match=r"2 slot\(s\) procesando"): + await llamacpp_service.stop_llamacpp(provider) + + process_mock.assert_not_called() + process_iter_mock.assert_not_called() + + +@pytest.mark.asyncio +async def test_stop_llamacpp_force_kills_busy_server( + tmp_path, + provider_factory, +): + provider = provider_factory() + status = {"running": True, "busy_slots": 2} + pid_file = tmp_path / "llamacpp.pid" + pid_file.write_text("4321", encoding="utf-8") + + with ( + patch.object( + llamacpp_service, + "get_status", + AsyncMock(return_value=status), + ), + patch.object(llamacpp_service, "_read_pid", return_value=4321), + patch.object(llamacpp_service, "_pid_file", return_value=pid_file), + patch.object(llamacpp_service.psutil, "Process") as process_mock, + patch.object( + llamacpp_service.psutil, + "process_iter", + return_value=[], + ) as process_iter_mock, + ): + result = await llamacpp_service.stop_llamacpp(provider, force=True) + + assert result == {"stopped": True, "killed": 1} + process_mock.assert_called_once_with(4321) + process_mock.return_value.kill.assert_called_once_with() + process_iter_mock.assert_called_once_with(["pid", "name", "cmdline"]) + assert not pid_file.exists() diff --git a/backend/tests/test_messages_native.py b/backend/tests/test_messages_native.py new file mode 100644 index 0000000..d69ae18 --- /dev/null +++ b/backend/tests/test_messages_native.py @@ -0,0 +1,97 @@ +"""Tests de la rama anthropic_native de /v1/messages (passthrough verbatim).""" +import json +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from fastapi.testclient import TestClient + +from app.models.provider import Provider + + +@pytest.fixture +def client(): + from app.main import app + from app.core.config import get_settings + api_key = get_settings().ui_api_key + return TestClient(app, headers={"x-api-key": api_key}) + + +def _native_provider(api_base: str = "http://127.0.0.1:4002") -> Provider: + return Provider( + id="llamacpp", + name="llama.cpp", + api_base=api_base, + litellm_prefix="openai", + active_model="qwen3-coder-next", + anthropic_native=True, + ) + + +async def _alines(lines): + for line in lines: + yield line + + +def _mock_stream(lines): + stream = AsyncMock() + stream.__aenter__ = AsyncMock(return_value=stream) + stream.__aexit__ = AsyncMock(return_value=False) + stream.status_code = 200 + stream.aiter_lines = lambda: _alines(lines) + return stream + + +def _post_messages(client, provider): + body = { + "model": "claude-sonnet-4-6", + "messages": [{"role": "user", "content": "hola"}], + "max_tokens": 50, + } + lines = [ + 'data: {"type": "message_start", "message": {"usage": {"input_tokens": 3}}}', + "", + 'data: {"type": "message_stop"}', + ] + with patch( + "app.api.messages.providers_service.get_active_provider", + return_value=provider, + ), patch("app.api.messages.httpx.AsyncClient") as mock_client: + instance = mock_client.return_value + instance.__aenter__ = AsyncMock(return_value=instance) + instance.__aexit__ = AsyncMock(return_value=False) + instance.stream = MagicMock(return_value=_mock_stream(lines)) + resp = client.post("/v1/messages", json=body) + return resp, instance.stream + + +def test_native_forwards_to_provider_messages_url(client): + resp, stream_mock = _post_messages(client, _native_provider()) + assert resp.status_code == 200 + _, args, kwargs = stream_mock.mock_calls[0] + assert args[0] == "POST" + assert args[1] == "http://127.0.0.1:4002/v1/messages" + + +def test_native_rewrites_model_to_active_model(client): + _, stream_mock = _post_messages(client, _native_provider()) + _, _, kwargs = stream_mock.mock_calls[0] + assert kwargs["json"]["model"] == "qwen3-coder-next" + assert kwargs["json"]["messages"][0]["content"] == "hola" + + +def test_native_dedupes_v1_suffix_in_api_base(client): + _, stream_mock = _post_messages(client, _native_provider("http://127.0.0.1:1234/v1")) + _, args, _ = stream_mock.mock_calls[0] + assert args[1] == "http://127.0.0.1:1234/v1/messages" + + +def test_native_relays_sse_lines(client): + resp, _ = _post_messages(client, _native_provider()) + assert "message_stop" in resp.text + + +def test_native_no_auth_header_without_env_var(client): + _, stream_mock = _post_messages(client, _native_provider()) + _, _, kwargs = stream_mock.mock_calls[0] + assert "Authorization" not in kwargs["headers"] + assert "x-api-key" not in kwargs["headers"] diff --git a/docs/superpowers/specs/2026-08-18-local-llm-multigpu-provider-design.md b/docs/superpowers/specs/2026-08-18-local-llm-multigpu-provider-design.md new file mode 100644 index 0000000..e27d448 --- /dev/null +++ b/docs/superpowers/specs/2026-08-18-local-llm-multigpu-provider-design.md @@ -0,0 +1,83 @@ +# Diseño: Proveedor LLM local multi-GPU (llama.cpp gestionado) + passthrough Anthropic nativo + +Fecha: 2026-08-18 +Estado: aprobado por autonomía delegada ("tienes toda la libertad") — pendiente de revisión posterior del usuario. + +## Contexto + +El usuario quiere que bipolar-code reemplace correctamente el backend de Claude Code en VS Code, sirviendo un modelo local grande sobre **dos GPUs AMD simultáneas** (Radeon AI PRO R9700 32GB + RX 7800 XT 16GB, ~48GB VRAM combinados) en este PC, y consumible desde PCs externos en la LAN. + +Investigación (last30days, 2026-08-18, guardada en `~/Documents/Last30Days/running-large-local-llms-on-dual-amd-gpus-r9700-and-rx-7800-xt-for-agentic-coding-raw-v3.md`): + +- **llama.cpp `llama-server` ya habla la Anthropic Messages API (`/v1/messages`) de forma nativa** (igual que Ollama desde enero 2026 y LM Studio ≥0.4.1). Claude Code funciona contra estos servidores con solo `ANTHROPIC_BASE_URL`, sin proxy de traducción. +- **Vulkan es la ruta multi-GPU heterogénea** (RDNA4 gfx1201 + RDNA3 gfx1101 mezcladas): `--split-mode layer` + `--tensor-split` manual proporcional. No hay P2P entre GPUs distintas; cada handoff paga GPU→PCIe→RAM→PCIe (por eso layer-split, no row-split). +- En R9700, RADV/Vulkan gana a ROCm en la mayoría de cargas llama.cpp (Phoronix); en Windows el backend Vulkan es la opción estable para ambas GPUs. +- **Modelos objetivo para el tier 48GB**: Qwen3-Coder-Next 80B-A3B (~35-40GB Q4, #1 SWE-bench local), GLM-series, Qwen3-Coder-30B (cabe en la R9700 sola). GLM-4.5-Air 4bit (~59GB) no cabe. +- Comunidad activa haciendo exactamente esto: r/LocalLLaMA "2× Radeon R9700 for Local AI" (52 comentarios), r/ROCm Qwen3.8-27B en R9700 (77 pts). + +## Decisiones (tomadas en autonomía; revisar si algo no cuadra) + +| # | Decisión | Justificación | +|---|---|---| +| D1 | Runtime local = **llama.cpp `llama-server` gestionado por bipolar-code** (spawn/kill/status, como ya se hace con litellm) | Único runtime con Vulkan hetero multi-GPU en Windows + `/v1/messages` nativo | +| D2 | Nuevo flag `anthropic_native` en `Provider`: si está activo, `/v1/messages` reenvía el body Anthropic **verbatim** al provider (tercer camino: ni litellm, ni traducción OAI) | Fidelidad total de tool-use/imágenes/streaming; menos código en el hot path | +| D3 | `--tensor-split` **auto proporcional a VRAM libre detectada** por GPU (parse de `llama-server --list-devices`), con override manual | Aprendizaje Upflow: admisión por capacidad real medida; ratios relativos, nunca constantes absolutas sobre magnitudes variables | +| D4 | Chequeo de ajuste (tamaño GGUF + estimación KV vs VRAM libre) = **warning, no bloqueo** | Aprendizaje Upflow: un clasificador con fallback nunca sirve como gate | +| D5 | Antes de matar `llama-server`, consultar in-flight (`/health`, slots); si hay trabajo activo, requerir `force=true` | Aprendizaje: "check active job before killing server" (corté un job RIFE real por no hacerlo) | +| D6 | Alcance = **single-box multi-GPU**. Distribuido entre PCs (llama.cpp RPC) → backlog v3. Descarga de modelos HF → backlog v3 (el usuario apunta a un GGUF local) | YAGNI; el caso externo-PC se cubre consumiendo bipolar-code por LAN | +| D7 | PCs externos: ya cubierto por `host=0.0.0.0` + auth por API key existentes; se añade panel "Conexión externa" en Settings con snippet copy-paste (`ANTHROPIC_BASE_URL` + key) | Todo el mecanismo ya existe en la rama v2; solo falta descubribilidad | + +## Arquitectura + +### Backend + +1. **`models/provider.py`** — dos campos nuevos: + - `anthropic_native: bool = False` — el provider expone `/v1/messages` nativo. + - `local_launch: dict = {}` — config de lanzamiento para providers locales gestionados: `{exe_path, model_path, ctx_size, split_mode, tensor_split ("auto" | [float]), ngl, extra_args}`. + +2. **`services/llamacpp_service.py`** (nuevo, ~250 líneas) — espejo del patrón litellm: + - `list_devices()` → parsea `llama-server --list-devices`: `[{index, backend, name, vram_total_mib, vram_free_mib}]`. + - `compute_tensor_split(devices)` → ratios proporcionales a VRAM libre. + - `estimate_fit(model_path, ctx_size, devices)` → `{fits: bool, needed_mib, available_mib}` (tamaño archivo × 1.15 + KV aprox; solo warning). + - `start_llamacpp(provider)` / `stop_llamacpp(force)` / `get_status()` — PID file `llamacpp.pid`, logs `llamacpp-{out,err}.log`, kill por PID + fallback cmdline, chequeo in-flight antes de matar. + - Puerto derivado del `api_base` del provider (default `http://127.0.0.1:4002`). + - Flags generados: `--model`, `--ctx-size`, `--n-gpu-layers`, `--split-mode layer`, `--tensor-split`, `--jinja`, `--host`, `--port`. + +3. **`api/llamacpp.py`** (nuevo router, thin): `GET /api/llamacpp/devices`, `GET /api/llamacpp/status`, `POST /api/llamacpp/start`, `POST /api/llamacpp/stop?force=`. + +4. **`api/messages.py`** — rama nueva al inicio de `messages_passthrough`: si `active.anthropic_native` → reescribir `body["model"] = active.active_model` y reenviar streaming/non-streaming a `{api_base}/v1/messages` sin transformar. Registro de usage igual que hoy (el response Anthropic trae `usage`). + +5. **`providers_service._DEFAULTS`** — nueva entrada `llamacpp` (`api_base=http://127.0.0.1:4002`, `anthropic_native=True`, `litellm_prefix="openai"` como fallback si algún flujo pasa por litellm). Switch a este provider **no** reinicia litellm si el server local ya está arriba; el arranque de llama-server es explícito vía UI (los arranques cargan 30-40GB a VRAM, no deben ser efecto colateral). + +### Frontend + +- `types/provider.ts`: `anthropic_native`, `local_launch`, tipos de devices/status. +- `services/llamacpp.ts` + `hooks/useLlamaCpp.ts` (TanStack Query). +- `components/LlamaCppPanel.tsx`: visible en Providers cuando `provider.id === "llamacpp"` — lista de GPUs con VRAM, path del GGUF, ctx, tensor-split auto/manual, botones Start/Stop, estado y warning de ajuste. +- `pages/Settings.tsx`: sección "Conexión externa" — muestra IP LAN + puerto + `ui_api_key` como snippet `ANTHROPIC_BASE_URL/ANTHROPIC_API_KEY` copy-paste para otros PCs. + +### Flujo de request (provider llamacpp activo) + +``` +Claude Code (este PC u otro) → FastAPI :8000 /v1/messages (auth API key) + → passthrough verbatim → llama-server :4002 /v1/messages (Vulkan, 2 GPUs tensor-split) + → SSE de vuelta sin transformar +``` + +## Manejo de errores + +- llama-server caído + provider llamacpp activo → 503 con mensaje accionable ("Inicia el servidor local desde Providers"). +- `--list-devices` falla o exe no encontrado → devices=[], UI muestra cómo instalar llama.cpp (release Vulkan de GitHub) y campo exe_path. +- Kill con in-flight sin `force` → 409 con detalle de slots ocupados. + +## Testing + +- `test_llamacpp_service.py`: parse de `--list-devices` (fixtures de output real), ratios de tensor-split (proporcionalidad, suma=1, redondeo), estimate_fit en límites, generación de cmdline. +- `test_messages_native_passthrough.py`: rama nativa reenvía body verbatim con model reescrito, streaming SSE se retransmite, usage se registra, 503 si server caído. +- Existentes de bypass no deben romperse. + +## Fuera de alcance (backlog v3) + +- llama.cpp RPC multi-host (juntar GPUs de PCs distintos). +- Descarga/gestión de modelos GGUF desde HF. +- Auto-arranque de llama-server al boot. diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 28f69d5..349ed68 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "bipolar-code-frontend", - "version": "0.1.0", + "version": "2.10.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "bipolar-code-frontend", - "version": "0.1.0", + "version": "2.10.0", "dependencies": { "@tanstack/react-query": "^5.40.0", "axios": "^1.7.2", diff --git a/frontend/package.json b/frontend/package.json index 2c69a7b..cef4a33 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1 +1 @@ -{"name":"bipolar-code-frontend","private":true,"version":"0.1.0","type":"module","scripts":{"dev":"vite","build":"tsc -b && vite build","preview":"vite preview","test":"vitest run"},"dependencies":{"@tanstack/react-query":"^5.40.0","axios":"^1.7.2","react":"^18.3.1","react-dom":"^18.3.1","react-router-dom":"^6.23.1","recharts":"^3.8.1"},"devDependencies":{"@testing-library/jest-dom":"^6.4.6","@testing-library/react":"^16.0.0","@types/react":"^18.3.3","@types/react-dom":"^18.3.0","@vitejs/plugin-react":"^4.3.1","autoprefixer":"^10.4.19","jsdom":"^29.0.2","postcss":"^8.4.39","tailwindcss":"^3.4.4","typescript":"^5.4.5","vite":"^5.3.1","vitest":"^1.6.0"}} \ No newline at end of file +{"name":"bipolar-code-frontend","private":true,"version":"2.10.0","type":"module","scripts":{"dev":"vite","build":"tsc -b && vite build","preview":"vite preview","test":"vitest run"},"dependencies":{"@tanstack/react-query":"^5.40.0","axios":"^1.7.2","react":"^18.3.1","react-dom":"^18.3.1","react-router-dom":"^6.23.1","recharts":"^3.8.1"},"devDependencies":{"@testing-library/jest-dom":"^6.4.6","@testing-library/react":"^16.0.0","@types/react":"^18.3.3","@types/react-dom":"^18.3.0","@vitejs/plugin-react":"^4.3.1","autoprefixer":"^10.4.19","jsdom":"^29.0.2","postcss":"^8.4.39","tailwindcss":"^3.4.4","typescript":"^5.4.5","vite":"^5.3.1","vitest":"^1.6.0"}} \ No newline at end of file diff --git a/frontend/src/components/LlamaCppPanel.tsx b/frontend/src/components/LlamaCppPanel.tsx new file mode 100644 index 0000000..fd4f8ff --- /dev/null +++ b/frontend/src/components/LlamaCppPanel.tsx @@ -0,0 +1,175 @@ +import { useState } from 'react' +import { useMutation, useQueryClient } from '@tanstack/react-query' +import { Badge } from '@/components/Badge' +import { Button } from '@/components/Button' +import { ModelDownloader } from '@/components/ModelDownloader' +import { providersApi } from '@/services/api' +import { useLlamaDevices, useLlamaLogs, useLlamaStatus, useStartLlama, useStopLlama } from '@/hooks/useLlamaCpp' +import type { Provider, LocalLaunchConfig } from '@/types/provider' + +interface LlamaCppPanelProps { + provider: Provider +} + +function formatGiB(mib: number): string { + return `${(mib / 1024).toFixed(1)} GiB` +} + +export function LlamaCppPanel({ provider }: LlamaCppPanelProps) { + const qc = useQueryClient() + const { data: devicesResp } = useLlamaDevices() + const { data: status } = useLlamaStatus() + const startLlama = useStartLlama() + const stopLlama = useStopLlama() + const [confirmForce, setConfirmForce] = useState(false) + const [showModels, setShowModels] = useState(false) + const [showLogs, setShowLogs] = useState(false) + const { data: logsData } = useLlamaLogs(showLogs) + + const launch: LocalLaunchConfig = provider.local_launch ?? {} + const [modelPath, setModelPath] = useState(launch.model_path ?? '') + const [exePath, setExePath] = useState(launch.exe_path ?? '') + const [ctxSize, setCtxSize] = useState(launch.ctx_size ?? 32768) + + const saveLaunch = useMutation({ + mutationFn: (updates: LocalLaunchConfig) => + providersApi.update(provider.id, { local_launch: { ...launch, ...updates } }), + onSuccess: () => qc.invalidateQueries({ queryKey: ['providers'] }), + }) + + const handleStop = () => { + stopLlama.mutate(confirmForce, { + onError: (err: unknown) => { + const isBusy = (err as { response?: { status?: number } })?.response?.status === 409 + if (isBusy) setConfirmForce(true) + }, + onSuccess: () => setConfirmForce(false), + }) + } + + const devices = devicesResp?.devices ?? [] + const totalFree = devices.reduce((acc, d) => acc + d.vram_free_mib, 0) + + return ( +
+
+ Servidor local + {status?.running ? ( + + ) : ( + + )} + {status?.running && status.busy_slots > 0 && ( + + )} +
+ {!status?.running ? ( + + ) : ( + + )} +
+
+ + {confirmForce && ( +

+ Hay requests en curso. Pulsa de nuevo para detener de todos modos. +

+ )} + + {startLlama.isError && ( +

+ {(startLlama.error as { response?: { data?: { detail?: string } } })?.response?.data?.detail || 'Error al iniciar'} +

+ )} + + {devicesResp && !devicesResp.exe_found && ( +

+ llama-server no encontrado. Descarga un release Vulkan de llama.cpp + (github.com/ggml-org/llama.cpp/releases) y configura la ruta abajo. +

+ )} + + {devices.length > 0 && ( +
+

+ GPUs detectadas — VRAM libre total: {formatGiB(totalFree)} (tensor-split auto proporcional) +

+ {devices.map((d) => ( +
+ + {d.name} + + {formatGiB(d.vram_free_mib)} libre / {formatGiB(d.vram_total_mib)} + +
+ ))} +
+ )} + +
+ +
+ + +
+ +
+ +
+ + +
+ + {showModels && } + + {showLogs && ( + 
+          {(logsData?.logs ?? []).join('\n') || 'Sin logs todavía'}
+
+ )} +
+ ) +} diff --git a/frontend/src/components/ModelDownloader.tsx b/frontend/src/components/ModelDownloader.tsx new file mode 100644 index 0000000..8e80e7c --- /dev/null +++ b/frontend/src/components/ModelDownloader.tsx @@ -0,0 +1,154 @@ +import { useState } from 'react' +import { useQuery } from '@tanstack/react-query' +import { Button } from '@/components/Button' +import { Badge } from '@/components/Badge' +import { llamacppApi } from '@/services/api' +import { useHFDownloads, useLocalModels, useStartDownload, useUseModel } from '@/hooks/useLlamaCpp' + +function formatSize(bytes: number): string { + if (bytes >= 1024 ** 3) return `${(bytes / 1024 ** 3).toFixed(1)} GB` + if (bytes >= 1024 ** 2) return `${(bytes / 1024 ** 2).toFixed(0)} MB` + return `${bytes} B` +} + +export function ModelDownloader({ activeModelPath }: { activeModelPath: string }) { + const [query, setQuery] = useState('') + const [searchTerm, setSearchTerm] = useState('') + const [selectedRepo, setSelectedRepo] = useState(null) + + const search = useQuery({ + queryKey: ['llamacpp', 'hf-search', searchTerm], + queryFn: () => llamacppApi.searchHF(searchTerm), + enabled: searchTerm.length > 1, + staleTime: 60_000, + }) + const files = useQuery({ + queryKey: ['llamacpp', 'hf-files', selectedRepo], + queryFn: () => llamacppApi.listHFFiles(selectedRepo!), + enabled: !!selectedRepo, + staleTime: 60_000, + }) + const { data: downloadsData } = useHFDownloads() + const { data: localData } = useLocalModels() + const startDownload = useStartDownload() + const useModel = useUseModel() + + const downloads = downloadsData?.downloads ?? [] + const active = downloads.filter(d => d.status === 'downloading' || d.status === 'queued') + + return ( +
+
+ setQuery(e.target.value)} + onKeyDown={(e) => e.key === 'Enter' && setSearchTerm(query.trim())} + placeholder="Buscar GGUF en Hugging Face (ej: Qwen3-Coder-Next)" + className="flex-1 text-xs border border-gray-300 rounded-lg px-2 py-1.5 focus:outline-none focus:ring-2 focus:ring-brand-400" + /> + +
+ + {search.data && !selectedRepo && ( +
+ {search.data.results.map(r => ( + + ))} + {search.data.results.length === 0 && ( +

Sin resultados

+ )} +
+ )} + + {selectedRepo && ( +
+
+ + {selectedRepo} +
+
+ {files.data?.files.map(f => ( +
+ {f.filename} + {f.parts.length > 1 && } + {formatSize(f.size)} + +
+ ))} + {files.isLoading &&

Cargando archivos…

} +
+
+ )} + + {active.length > 0 && ( +
+ {active.map(d => { + const pct = d.total_bytes > 0 ? Math.min(100, (d.downloaded_bytes / d.total_bytes) * 100) : 0 + return ( +
+
+ {d.filename} + + {formatSize(d.downloaded_bytes)} / {formatSize(d.total_bytes)} + {d.speed_bps > 0 && ` — ${formatSize(d.speed_bps)}/s`} + + +
+
+
+
+
+ ) + })} +
+ )} + + {downloads.filter(d => d.status === 'error').map(d => ( +

+ Error descargando {d.filename}: {d.error} +

+ ))} + + {(localData?.models.length ?? 0) > 0 && ( +
+

Modelos locales

+ {localData!.models.map(m => ( +
+ {m.filename} + {activeModelPath === m.path && } + {formatSize(m.size)} + {activeModelPath !== m.path && ( + + )} +
+ ))} +
+ )} +
+ ) +} diff --git a/frontend/src/hooks/useLlamaCpp.ts b/frontend/src/hooks/useLlamaCpp.ts new file mode 100644 index 0000000..7641e2b --- /dev/null +++ b/frontend/src/hooks/useLlamaCpp.ts @@ -0,0 +1,79 @@ +import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query' +import { llamacppApi } from '@/services/api' + +export function useLlamaDevices(enabled = true) { + return useQuery({ + queryKey: ['llamacpp', 'devices'], + queryFn: llamacppApi.getDevices, + enabled, + staleTime: 30_000, + }) +} + +export function useLlamaStatus(enabled = true) { + return useQuery({ + queryKey: ['llamacpp', 'status'], + queryFn: llamacppApi.getStatus, + enabled, + refetchInterval: 5_000, + }) +} + +export function useStartLlama() { + const qc = useQueryClient() + return useMutation({ + mutationFn: () => llamacppApi.start(), + onSuccess: () => qc.invalidateQueries({ queryKey: ['llamacpp', 'status'] }), + }) +} + +export function useStopLlama() { + const qc = useQueryClient() + return useMutation({ + mutationFn: (force: boolean) => llamacppApi.stop(force), + onSuccess: () => qc.invalidateQueries({ queryKey: ['llamacpp', 'status'] }), + }) +} + +export function useLlamaLogs(enabled: boolean) { + return useQuery({ + queryKey: ['llamacpp', 'logs'], + queryFn: () => llamacppApi.getLogs(120), + enabled, + refetchInterval: 3_000, + }) +} + +export function useHFDownloads(enabled = true) { + return useQuery({ + queryKey: ['llamacpp', 'downloads'], + queryFn: llamacppApi.getDownloads, + enabled, + refetchInterval: 2_000, + }) +} + +export function useLocalModels() { + return useQuery({ + queryKey: ['llamacpp', 'local-models'], + queryFn: llamacppApi.getLocalModels, + staleTime: 10_000, + }) +} + +export function useStartDownload() { + const qc = useQueryClient() + return useMutation({ + mutationFn: ({ repoId, filename }: { repoId: string; filename: string }) => + llamacppApi.download(repoId, filename), + onSuccess: () => qc.invalidateQueries({ queryKey: ['llamacpp', 'downloads'] }), + }) +} + +export function useUseModel() { + const qc = useQueryClient() + return useMutation({ + mutationFn: (path: string) => llamacppApi.useModel(path), + onSuccess: () => qc.invalidateQueries({ queryKey: ['providers'] }), + }) +} diff --git a/frontend/src/pages/Providers.tsx b/frontend/src/pages/Providers.tsx index 6a37ce5..cceb189 100644 --- a/frontend/src/pages/Providers.tsx +++ b/frontend/src/pages/Providers.tsx @@ -5,6 +5,7 @@ import { Button } from '@/components/Button' import { Spinner } from '@/components/Spinner' import { AddProviderModal } from '@/components/AddProviderModal' import { NvidiaWizard } from '@/components/NvidiaWizard' +import { LlamaCppPanel } from '@/components/LlamaCppPanel' import { useProviders, useSwitchProvider, useDeleteProvider } from '@/hooks/useProviders' import { useQuery } from '@tanstack/react-query' import { settingsApi } from '@/services/api' @@ -116,6 +117,7 @@ export function Providers() { )}
+ {p.id === 'llamacpp' && } ) })} diff --git a/frontend/src/pages/Settings.tsx b/frontend/src/pages/Settings.tsx index 312c117..b71eac7 100644 --- a/frontend/src/pages/Settings.tsx +++ b/frontend/src/pages/Settings.tsx @@ -68,9 +68,13 @@ export function Settings() { } | null>(null) const [fullKey, setFullKey] = useState(null) const [copied, setCopied] = useState(false) + const [lanUrl, setLanUrl] = useState(null) useEffect(() => { settingsApi.getAuthInfo().then(setAuthInfo).catch(() => {}) + settingsApi.getConnectionInfo() + .then(info => setLanUrl(info.anthropic_base_url || null)) + .catch(() => {}) }, []) const loadAndCopyKey = useCallback(async () => { @@ -175,7 +179,7 @@ export function Settings() { Configura estas variables en ~/.claude/settings.json de cada PC:

-
ANTHROPIC_BASE_URL = http://<ip-servidor>:8000
+
ANTHROPIC_BASE_URL = {lanUrl ?? 'http://:8000'}
ANTHROPIC_API_KEY ={' '} {fullKey ?? authInfo?.api_key_prefix ?? '…'} diff --git a/frontend/src/services/api.ts b/frontend/src/services/api.ts index 436245c..7489f56 100644 --- a/frontend/src/services/api.ts +++ b/frontend/src/services/api.ts @@ -1,6 +1,9 @@ import axios from 'axios' import type { ModelEntry, UsageStats } from '@/types' -import type { Provider, ProviderRegistry, ProviderModel } from '@/types/provider' +import type { + Provider, ProviderRegistry, ProviderModel, LlamaDevicesResponse, LlamaStatus, + HFRepo, HFFile, HFDownload, LocalModel, +} from '@/types/provider' const STORAGE_KEY = 'bipolar_api_key' @@ -86,6 +89,27 @@ export const settingsApi = { allowed_origins: string }>('/settings/auth-info').then(r => r.data), getApiKey: () => api.get<{ api_key: string }>('/settings/api-key').then(r => r.data), + getConnectionInfo: () => api.get<{ + lan_ip: string + port: number + anthropic_base_url: string + }>('/settings/connection-info').then(r => r.data), +} + +export const llamacppApi = { + getDevices: () => api.get('/llamacpp/devices').then(r => r.data), + getStatus: () => api.get('/llamacpp/status').then(r => r.data), + start: () => api.post('/llamacpp/start').then(r => r.data), + stop: (force = false) => api.post('/llamacpp/stop', null, { params: { force } }).then(r => r.data), + getLogs: (lines = 80) => api.get<{ logs: string[] }>('/llamacpp/logs', { params: { lines } }).then(r => r.data), + searchHF: (q: string) => api.get<{ results: HFRepo[] }>('/llamacpp/hf/search', { params: { q } }).then(r => r.data), + listHFFiles: (repoId: string) => api.get<{ files: HFFile[] }>('/llamacpp/hf/files', { params: { repo_id: repoId } }).then(r => r.data), + download: (repoId: string, filename: string) => + api.post('/llamacpp/hf/download', { repo_id: repoId, filename }).then(r => r.data), + getDownloads: () => api.get<{ downloads: HFDownload[] }>('/llamacpp/hf/downloads').then(r => r.data), + cancelDownload: (id: string) => api.delete(`/llamacpp/hf/download/${encodeURIComponent(id)}`).then(r => r.data), + getLocalModels: () => api.get<{ models: LocalModel[] }>('/llamacpp/models').then(r => r.data), + useModel: (path: string) => api.post('/llamacpp/use-model', { path }).then(r => r.data), } export const usageApi = { diff --git a/frontend/src/types/provider.ts b/frontend/src/types/provider.ts index 5b65566..b8d85e3 100644 --- a/frontend/src/types/provider.ts +++ b/frontend/src/types/provider.ts @@ -14,6 +14,73 @@ export interface Provider { use_chat_completions_for_anthropic: boolean max_tools: number rate_limit_rpm: number + anthropic_native?: boolean + local_launch?: LocalLaunchConfig +} + +export interface LocalLaunchConfig { + exe_path?: string + model_path?: string + ctx_size?: number + split_mode?: string + tensor_split?: 'auto' | number[] + ngl?: number + extra_args?: string[] + autostart?: boolean +} + +export interface LlamaDevice { + index: number + backend: string + name: string + vram_total_mib: number + vram_free_mib: number +} + +export interface LlamaDevicesResponse { + exe_found: boolean + exe_path: string + devices: LlamaDevice[] +} + +export interface LlamaStatus { + running: boolean + pid: number | null + port: number | null + model_path: string | null + healthy: boolean + busy_slots: number +} + +export interface HFRepo { + id: string + downloads: number + likes: number + updated: string +} + +export interface HFFile { + filename: string + size: number + parts: string[] +} + +export interface HFDownload { + id: string + repo_id: string + filename: string + parts: string[] + status: 'queued' | 'downloading' | 'done' | 'error' | 'cancelled' + total_bytes: number + downloaded_bytes: number + speed_bps: number + error: string +} + +export interface LocalModel { + filename: string + path: string + size: number } export interface ProviderRegistry {