From 8551c27a7d150f14b0e1dd921713c3d191bbd580 Mon Sep 17 00:00:00 2001 From: huangzesen Date: Thu, 16 Jul 2026 02:06:13 -0700 Subject: [PATCH 1/2] feat(vision): honor current preset routes Route direct vision only through the current preset's provider, model, endpoint, credential, headers, and supported wire. Keep unsupported routes explicit/manual, remove the retired automatic MiniMax/Zhipu services, and ship provider-neutral manual guidance plus owning regressions. --- src/lingtai/kernel/base_agent/identity.py | 18 +- src/lingtai/services/vision/ANATOMY.md | 69 +- src/lingtai/services/vision/__init__.py | 33 +- src/lingtai/services/vision/anthropic.py | 6 +- src/lingtai/services/vision/codex.py | 5 +- src/lingtai/services/vision/gemini.py | 3 +- src/lingtai/services/vision/mimo.py | 3 +- src/lingtai/services/vision/minimax.py | 94 --- src/lingtai/services/vision/openai.py | 30 +- src/lingtai/services/vision/zhipu.py | 87 --- src/lingtai/tools/vision/ANATOMY.md | 58 +- src/lingtai/tools/vision/CONTRACT.md | 186 ++--- src/lingtai/tools/vision/__init__.py | 326 +++++++-- src/lingtai/tools/vision/glossary-wen.md | 2 + src/lingtai/tools/vision/glossary-zh.md | 2 + src/lingtai/tools/vision/manual/SKILL.md | 42 ++ tests/test_agent_capabilities.py | 23 +- tests/test_agent_preset_manifest.py | 18 +- tests/test_tools_package_data.py | 4 + tests/test_validate_skill.py | 10 +- tests/test_vision_capability.py | 817 ++++++++++++++++++++-- tests/test_vision_services.py | 70 ++ 22 files changed, 1365 insertions(+), 541 deletions(-) delete mode 100644 src/lingtai/services/vision/minimax.py delete mode 100644 src/lingtai/services/vision/zhipu.py create mode 100644 src/lingtai/tools/vision/manual/SKILL.md diff --git a/src/lingtai/kernel/base_agent/identity.py b/src/lingtai/kernel/base_agent/identity.py index 14a5f0659..9fec6e58f 100644 --- a/src/lingtai/kernel/base_agent/identity.py +++ b/src/lingtai/kernel/base_agent/identity.py @@ -8,6 +8,22 @@ import os import platform import sys +from urllib.parse import SplitResult, urlsplit, urlunsplit + + +def sanitize_endpoint(value: str) -> str: + """Remove endpoint credentials, query parameters, and fragments for display.""" + try: + parsed = urlsplit(value) + if not parsed.scheme or not parsed.netloc: + return "" + hostname = parsed.hostname or "" + host = f"[{hostname}]" if ":" in hostname and not hostname.startswith("[") else hostname + if parsed.port is not None: + host = f"{host}:{parsed.port}" + return urlunsplit(SplitResult(parsed.scheme, host, parsed.path, "", "")) + except (TypeError, ValueError): + return "" def _set_name(agent, name: str) -> None: @@ -125,7 +141,7 @@ def _safe_llm_from_service(agent) -> dict: base_url = _effective_base_url_from_service(service) if isinstance(base_url, str) and base_url: - llm["base_url"] = base_url + llm["base_url"] = sanitize_endpoint(base_url) context_limit = _safe_int_attr(service, "_context_window") if context_limit is not None: diff --git a/src/lingtai/services/vision/ANATOMY.md b/src/lingtai/services/vision/ANATOMY.md index 5a49888ef..afd3269df 100644 --- a/src/lingtai/services/vision/ANATOMY.md +++ b/src/lingtai/services/vision/ANATOMY.md @@ -1,71 +1,60 @@ --- related_files: - - src/lingtai/tools/vision/ANATOMY.md - src/lingtai/services/ANATOMY.md + - src/lingtai/tools/vision/ANATOMY.md - src/lingtai/services/vision/__init__.py - src/lingtai/services/vision/anthropic.py - src/lingtai/services/vision/codex.py - src/lingtai/services/vision/gemini.py - src/lingtai/services/vision/local.py - src/lingtai/services/vision/mimo.py - - src/lingtai/services/vision/minimax.py - src/lingtai/services/vision/openai.py - - src/lingtai/services/vision/zhipu.py maintenance: | Keep related_files as repo-relative paths to real files. Include neighboring ANATOMY.md files so the anatomy graph stays connected rather than isolated; - anatomy links must be bidirectional. If you create a new ANATOMY.md, copy this - maintenance field. If you notice drift between this anatomy and the code, - report it. See lingtai-dev-guide for details. + anatomy links must be bidirectional. If code or citations drift, update this + map with the code change and run the architecture/document checks. --- # src/lingtai/services/vision/ -Provider-specific image understanding — standalone services that own their own API clients. - -> **Maintenance:** see the `lingtai-kernel-anatomy` skill. **Coding agents** update this file in the same commit as code changes. **LingTai agents** report drift as issues. +Standalone image-understanding services. Each service owns its SDK client and +credentials; `local` is an explicit on-device pseudo-provider. ## Components -| File | LOC | Role | -|------|-----|------| -| `__init__.py` | 112 | `VisionService` ABC, `_MIME_BY_EXT` map, `_read_image()` helper, `create_vision_service()` factory | -| `anthropic.py` | 61 | `AnthropicVisionService` — base64 inline image via Anthropic Messages API | -| `codex.py` | 78 | `CodexVisionService` — ChatGPT Codex Responses API vision via OAuth token | -| `gemini.py` | 53 | `GeminiVisionService` — `genai.Client` + `types.Part.from_bytes` | -| `local.py` | 72 | `LocalVisionService` — mlx-vlm on Apple Silicon, lazy model load | -| `mimo.py` | 75 | `MiMoVisionService` — OpenAI SDK to `api.xiaomimimo.com/v1` | -| `minimax.py` | 94 | `MiniMaxVisionService` — MCP `understand_image` tool via `minimax-coding-plan-mcp` | -| `openai.py` | 56 | `OpenAIVisionService` — OpenAI chat completions with `image_url` content part | -| `zhipu.py` | 87 | `ZhipuVisionService` — MCP `analyze_image` via `@z_ai/mcp-server` Node.js subprocess | +| File | Role | +|---|---| +| `__init__.py:18-79` | `VisionService`, MIME map, image readers, and shared OpenAI-compatible message builder | +| `__init__.py:82-125` | `create_vision_service()` lazy factory for `anthropic`, `openai`, `gemini`, `mimo`, `codex`, and `local` | +| `anthropic.py:9-70` | Anthropic Messages image service; accepts active model, endpoint, headers, and token limit | +| `openai.py:7-72` | OpenAI Chat Completions or Responses image service; preserves model, endpoint, headers, wire, and output limit | +| `mimo.py:26-59` | MiMo Chat Completions service; constructor accepts only API key, model, endpoint, and token limit | +| `gemini.py:7-53` | Gemini SDK image service | +| `codex.py:11-79` | Codex Responses service using OAuth token and current model/endpoint | +| `local.py:20-72` | Local mlx-vlm pseudo-provider with lazy model loading | ## Connections -- **ABC contract** — all providers inherit `VisionService` (`__init__.py:17`); single abstract method `analyze_image(image_path, prompt) -> str`. -- **Factory** — `create_vision_service(provider, api_key=...)` at `__init__.py:63` dispatches by name with lazy imports. Supported: `anthropic`, `openai`, `gemini`, `minimax`, `zhipu`, `mimo`, `codex`, `local`. -- **MCP dependency** — `minimax.py` and `zhipu.py` import `lingtai.services.mcp.MCPClient` for subprocess-based tool calls. -- **External SDKs** — `anthropic` (Anthropic SDK), `openai` (OpenAI SDK; OpenAI/MiMo/Codex), `google.genai` (Gemini), `mlx_vlm` (local). -- **Logging** — MCP providers use `lingtai.kernel.logging.get_logger`. +- `src/lingtai/tools/vision/__init__.py` imports the factory lazily during setup. +- API services read images through `_read_image()` and encode them as required by + their wire; local passes the file path to mlx-vlm. +- OpenAI Responses uses `input_text`/`input_image` and `max_output_tokens`; Codex + retains its separate streaming Responses request shape. ## Composition -- **Standalone ownership** — each service creates and owns its own SDK client + credentials. API providers use API keys; Codex uses `CodexTokenManager`/ChatGPT OAuth; all remain independent of LLM adapters or agents. -- **Shared helper** — `_read_image(image_path) -> (bytes, mime_type)` at `__init__.py:47` used by all API-based providers. -- **MCP lazy init** — `minimax.py:34` (`_ensure_client`) and `zhipu.py:29` (`_ensure_client`) start MCP subprocesses on first call, with stale-connection recovery. Both expose `close()` for subprocess teardown. -- **Local lazy load** — `local.py:39` (`_ensure_loaded`) defers `mlx_vlm.load()` to first `analyze_image` call. +The factory dispatches only to the six services named above. MiniMax is routed +by the capability layer to Anthropic, and compatible aliases are routed to the +OpenAI or Anthropic service; no MCP vision service remains here. ## State -- **Per-call stateless** — all services read the image file fresh each call, no caching. -- **Persistent MCP clients** — `minimax._client` and `zhipu._client` hold subprocess handles across calls. -- **Local model refs** — `local._model`, `local._processor`, `local._config` persist after first load. +Services keep their client/model configuration in memory. Local keeps its lazy +model references after first load; no service writes agent working-directory +state. ## Notes -- **Image encoding** — Anthropic (`anthropic.py:34`) and OpenAI/MiMo/Codex (`openai.py:38`, `mimo.py:57`, `codex.py:45`) use base64 data URLs. Gemini (`gemini.py:37`) uses `types.Part.from_bytes`. Local passes file path directly to mlx-vlm. MCP providers send base64 in tool args (minimax) or file path (zhipu). -- **Default models** — Anthropic: `claude-sonnet-4-20250514`; Gemini: `gemini-2.5-flash`; OpenAI: `gpt-4o`; MiMo: `mimo-v2.5`; Codex: `gpt-5.5`; Local: `mlx-community/paligemma2-3b-ft-docci-448-8bit`. -- **MCP tool names** — MiniMax: `understand_image` (`minimax.py:77`); Zhipu: `analyze_image` (`zhipu.py:70`). -- **MCP launchers** — MiniMax uses `uvx minimax-coding-plan-mcp -y` (`minimax.py:64`); Zhipu uses `npx -y @z_ai/mcp-server` (`zhipu.py:58`). -- **Zhipu path vs base64** — Zhipu MCP reads the file directly by path (`zhipu.py:70`), unlike other providers that base64-encode. -- **Codex Responses API** — `codex.py` constructs `OpenAI(base_url="https://chatgpt.com/backend-api/codex")`, passes `instructions`, `stream=True`, `store=False`, `input_text` + `input_image` content blocks, and concatenates `response.output_text.delta` events. It omits `max_output_tokens` by default because the live ChatGPT Codex backend rejected that parameter (`codex.py:25-28`, `codex.py:70-71`). -- **Gemini thought filtering** — `gemini.py:51` skips `part.text` when `part.thought` is True to exclude reasoning output. -- **Git history** — 7 commits. Key: MiMo provider addition (`a728864`), zhipu path-based input fix (`2e6d53c`), region-aware ZAI/ZHIPU mode (`bed1c1e`). +The capability layer supplies the active preset model and endpoint when the +route supports them. It supplies active headers to Anthropic/OpenAI where their +constructors accept them, and never forwards headers or wire metadata to MiMo. diff --git a/src/lingtai/services/vision/__init__.py b/src/lingtai/services/vision/__init__.py index 1a4e7508b..f69bf9f93 100644 --- a/src/lingtai/services/vision/__init__.py +++ b/src/lingtai/services/vision/__init__.py @@ -1,7 +1,7 @@ """VisionService — abstract image understanding backing the vision capability. -Provides standalone vision implementations for each provider that take their -own API key and handle file reading, base64 encoding, and SDK calls directly. +Provides standalone vision implementations for supported direct providers. +The local implementation is an explicit pseudo-provider and needs no API key. Usage: from lingtai.services.vision import VisionService, create_vision_service @@ -79,11 +79,21 @@ def _image_url_messages(image_path: str, prompt: str | None = None) -> list[dict ] +def _require_api_key(api_key: object, provider: str) -> str: + """Require a nonblank explicit API key without changing its value.""" + if not isinstance(api_key, str) or not api_key.strip(): + raise ValueError( + f"api_key is required for provider {provider!r}. " + f"Use provider='local' for on-device vision without an API key." + ) + return api_key + + def create_vision_service(provider: str, *, api_key: str | None = None, **kwargs) -> VisionService: """Factory — create a VisionService for the given provider. Args: - provider: Provider name ("anthropic", "openai", "gemini", "minimax", "codex", "local"). + provider: Provider name ("anthropic", "openai", "gemini", "mimo", "codex", "local"). api_key: API key for the provider (not required for "codex" or "local"). **kwargs: Additional provider-specific kwargs (e.g., model, base_url). @@ -97,14 +107,13 @@ def create_vision_service(provider: str, *, api_key: str | None = None, **kwargs from .local import LocalVisionService return LocalVisionService(**kwargs) elif provider == "codex": + token_path = kwargs.get("token_path") + if not isinstance(token_path, str) or not token_path.strip(): + raise ValueError("token_path is required for Codex vision.") from .codex import CodexVisionService return CodexVisionService(**kwargs) - if api_key is None: - raise ValueError( - f"api_key is required for provider {provider!r}. " - f"Use provider='local' for on-device vision without an API key." - ) + api_key = _require_api_key(api_key, provider) if provider == "anthropic": from .anthropic import AnthropicVisionService @@ -115,17 +124,11 @@ def create_vision_service(provider: str, *, api_key: str | None = None, **kwargs elif provider == "gemini": from .gemini import GeminiVisionService return GeminiVisionService(api_key=api_key, **kwargs) - elif provider == "minimax": - from .minimax import MiniMaxVisionService - return MiniMaxVisionService(api_key=api_key, **kwargs) - elif provider == "zhipu": - from .zhipu import ZhipuVisionService - return ZhipuVisionService(api_key=api_key, **kwargs) elif provider == "mimo": from .mimo import MiMoVisionService return MiMoVisionService(api_key=api_key, **kwargs) else: raise ValueError( f"Unsupported vision provider: {provider!r}. " - f"Supported: anthropic, openai, gemini, minimax, zhipu, mimo, codex, local." + f"Supported: anthropic, openai, gemini, mimo, codex, local." ) diff --git a/src/lingtai/services/vision/anthropic.py b/src/lingtai/services/vision/anthropic.py index 96ef2c075..ab989fc55 100644 --- a/src/lingtai/services/vision/anthropic.py +++ b/src/lingtai/services/vision/anthropic.py @@ -3,7 +3,7 @@ import base64 -from . import VisionService, _read_image +from . import VisionService, _read_image, _require_api_key class AnthropicVisionService(VisionService): @@ -20,7 +20,9 @@ def __init__( model: str = "claude-sonnet-4-20250514", base_url: str | None = None, max_tokens: int = 1024, + default_headers: dict | None = None, ) -> None: + api_key = _require_api_key(api_key, "anthropic") import anthropic as _anthropic client_kwargs: dict = {"api_key": api_key} @@ -28,6 +30,8 @@ def __init__( # anthropic-compat local proxies (e.g. JoyCodeProxy) need an explicit # endpoint; the SDK otherwise defaults to api.anthropic.com. client_kwargs["base_url"] = base_url + if default_headers: + client_kwargs["default_headers"] = dict(default_headers) self._client = _anthropic.Anthropic(**client_kwargs) self._model = model self._max_tokens = max_tokens diff --git a/src/lingtai/services/vision/codex.py b/src/lingtai/services/vision/codex.py index 7c0683a91..f2e578515 100644 --- a/src/lingtai/services/vision/codex.py +++ b/src/lingtai/services/vision/codex.py @@ -4,7 +4,6 @@ import base64 from typing import Any -from ...auth.codex import CodexTokenManager from . import VisionService, _read_image @@ -30,6 +29,10 @@ def __init__( token_path: str | None = None, **_ignored: Any, ) -> None: + if not isinstance(token_path, str) or not token_path.strip(): + raise ValueError("token_path is required for Codex vision.") + + from ...auth.codex import CodexTokenManager import openai as _openai self._openai = _openai diff --git a/src/lingtai/services/vision/gemini.py b/src/lingtai/services/vision/gemini.py index 14988f5c6..03325901a 100644 --- a/src/lingtai/services/vision/gemini.py +++ b/src/lingtai/services/vision/gemini.py @@ -1,7 +1,7 @@ """Gemini vision service — standalone image analysis via Google's genai SDK.""" from __future__ import annotations -from . import VisionService, _read_image +from . import VisionService, _read_image, _require_api_key class GeminiVisionService(VisionService): @@ -17,6 +17,7 @@ def __init__( api_key: str, model: str = "gemini-3-flash-preview", ) -> None: + api_key = _require_api_key(api_key, "gemini") from google import genai from google.genai import types diff --git a/src/lingtai/services/vision/mimo.py b/src/lingtai/services/vision/mimo.py index f41d70d83..d82924dd6 100644 --- a/src/lingtai/services/vision/mimo.py +++ b/src/lingtai/services/vision/mimo.py @@ -16,7 +16,7 @@ """ from __future__ import annotations -from . import VisionService, _image_url_messages +from . import VisionService, _image_url_messages, _require_api_key _MIMO_BASE_URL = "https://api.xiaomimimo.com/v1" @@ -38,6 +38,7 @@ def __init__( base_url: str | None = None, max_tokens: int = 1024, ) -> None: + api_key = _require_api_key(api_key, "mimo") import openai as _openai self._client = _openai.OpenAI( diff --git a/src/lingtai/services/vision/minimax.py b/src/lingtai/services/vision/minimax.py deleted file mode 100644 index a16a9577d..000000000 --- a/src/lingtai/services/vision/minimax.py +++ /dev/null @@ -1,94 +0,0 @@ -"""MiniMax vision service — standalone image analysis via MiniMax MCP understand_image tool.""" -from __future__ import annotations - -import base64 - -from . import VisionService, _read_image - -from lingtai.kernel.logging import get_logger - -logger = get_logger() - - -class MiniMaxVisionService(VisionService): - """Image understanding via MiniMax's ``understand_image`` MCP tool. - - Creates its own MCPClient for the ``minimax-coding-plan-mcp`` server. - The ``api_key`` is passed as ``MINIMAX_API_KEY`` in the subprocess env. - """ - - def __init__( - self, - *, - api_key: str, - api_host: str | None = None, - ) -> None: - self._api_key = api_key - if not api_host: - raise RuntimeError( - "api_host is required for MiniMax vision service." - ) - self._api_host = api_host - self._client = None # lazy init - - def _ensure_client(self): - """Lazily start the MCP client subprocess.""" - if self._client is not None: - from ...services.mcp import MCPClient - if self._client.is_connected(): - return - # Try to close stale client - try: - self._client.close() - except Exception: - pass - - import os - import shutil - from ...services.mcp import MCPClient - - uvx_path = shutil.which("uvx") - if not uvx_path: - raise RuntimeError( - "uvx not found. Please install uv: " - "https://docs.astral.sh/uv/getting-started/installation/" - ) - - env = { - **os.environ, - "MINIMAX_API_KEY": self._api_key, - "MINIMAX_API_HOST": self._api_host, - } - self._client = MCPClient( - command=uvx_path, - args=["minimax-coding-plan-mcp", "-y"], - env=env, - ) - self._client.start() - - def analyze_image(self, image_path: str, prompt: str | None = None) -> str: - """Analyze an image using MiniMax's understand_image MCP tool.""" - image_bytes, mime_type = _read_image(image_path) - question = prompt or "Describe this image." - - self._ensure_client() - - b64 = base64.b64encode(image_bytes).decode("ascii") - result = self._client.call_tool("understand_image", { - "image_source": f"data:{mime_type};base64,{b64}", - "prompt": question, - }) - if result.get("status") == "error": - msg = result.get("message", "unknown error") - logger.warning("MiniMax MCP vision error: %s", msg) - return "" - return result.get("text", "") - - def close(self) -> None: - """Shut down the MCP client subprocess.""" - if self._client is not None: - try: - self._client.close() - except Exception: - pass - self._client = None diff --git a/src/lingtai/services/vision/openai.py b/src/lingtai/services/vision/openai.py index 7f5854532..4639bac7f 100644 --- a/src/lingtai/services/vision/openai.py +++ b/src/lingtai/services/vision/openai.py @@ -1,7 +1,7 @@ """OpenAI vision service — standalone image analysis via OpenAI's multimodal API.""" from __future__ import annotations -from . import VisionService, _image_url_messages +from . import VisionService, _image_url_messages, _read_image, _require_api_key class OpenAIVisionService(VisionService): @@ -18,18 +18,46 @@ def __init__( model: str = "gpt-4o", base_url: str | None = None, max_tokens: int = 1024, + default_headers: dict | None = None, + wire_api: str = "chat_completions", ) -> None: + api_key = _require_api_key(api_key, "openai") + normalized_wire = wire_api.strip().lower() if isinstance(wire_api, str) else wire_api + if not isinstance(normalized_wire, str) or normalized_wire not in { + "chat_completions", + "responses", + }: + raise ValueError( + "Unsupported OpenAI vision wire; use vision(action='manual')." + ) + import openai as _openai kwargs: dict = {"api_key": api_key} if base_url: kwargs["base_url"] = base_url + if default_headers: + kwargs["default_headers"] = dict(default_headers) self._client = _openai.OpenAI(**kwargs) self._model = model self._max_tokens = max_tokens + self._wire_api = normalized_wire def analyze_image(self, image_path: str, prompt: str | None = None) -> str: """Analyze an image using OpenAI's vision capabilities.""" + if self._wire_api == "responses": + image_bytes, mime_type = _read_image(image_path) + import base64 + data_url = f"data:{mime_type};base64,{base64.b64encode(image_bytes).decode('utf-8')}" + raw = self._client.responses.create( + model=self._model, + max_output_tokens=self._max_tokens, + input=[{"role": "user", "content": [ + {"type": "input_text", "text": prompt or "Describe this image."}, + {"type": "input_image", "image_url": data_url}, + ]}], + ) + return getattr(raw, "output_text", "") or "" raw = self._client.chat.completions.create( model=self._model, messages=_image_url_messages(image_path, prompt), diff --git a/src/lingtai/services/vision/zhipu.py b/src/lingtai/services/vision/zhipu.py deleted file mode 100644 index 8cbab50d2..000000000 --- a/src/lingtai/services/vision/zhipu.py +++ /dev/null @@ -1,87 +0,0 @@ -"""Zhipu vision service — standalone image analysis via Z.AI MCP server.""" -from __future__ import annotations - -from . import VisionService - -from lingtai.kernel.logging import get_logger - -logger = get_logger() - - -class ZhipuVisionService(VisionService): - """Image understanding via Z.AI's ``image_analysis`` MCP tool. - - Creates its own MCPClient for the ``@z_ai/mcp-server`` Node.js server. - The ``api_key`` is passed as ``Z_AI_API_KEY`` in the subprocess env. - """ - - def __init__( - self, - *, - api_key: str, - z_ai_mode: str = "ZAI", - **_kwargs, - ) -> None: - self._api_key = api_key - self._z_ai_mode = z_ai_mode - self._client = None # lazy init - - def _ensure_client(self): - """Lazily start the MCP client subprocess.""" - if self._client is not None: - from ...services.mcp import MCPClient - if self._client.is_connected(): - return - try: - self._client.close() - except Exception: - pass - - import os - import shutil - from ...services.mcp import MCPClient - - npx_path = shutil.which("npx") - if not npx_path: - raise RuntimeError( - "npx not found. Please install Node.js >= v22: " - "https://nodejs.org/" - ) - - env = { - **os.environ, - "Z_AI_API_KEY": self._api_key, - "Z_AI_MODE": self._z_ai_mode, - } - self._client = MCPClient( - command=npx_path, - args=["-y", "@z_ai/mcp-server"], - env=env, - ) - self._client.start() - - def analyze_image(self, image_path: str, prompt: str | None = None) -> str: - """Analyze an image using Z.AI's analyze_image MCP tool.""" - question = prompt or "Describe this image." - - self._ensure_client() - - # The MCP server reads the file directly by path - result = self._client.call_tool("analyze_image", { - "image_source": image_path, - "prompt": question, - }) - if result.get("status") == "error": - msg = result.get("message", "unknown error") - logger.warning("Zhipu MCP vision error: %s", msg) - return "" - return result.get("text", "") - - def close(self) -> None: - """Shut down the MCP client subprocess.""" - if self._client is not None: - try: - self._client.close() - except Exception: - pass - self._client = None diff --git a/src/lingtai/tools/vision/ANATOMY.md b/src/lingtai/tools/vision/ANATOMY.md index 292c1a346..62fe87795 100644 --- a/src/lingtai/tools/vision/ANATOMY.md +++ b/src/lingtai/tools/vision/ANATOMY.md @@ -2,56 +2,52 @@ related_files: - src/lingtai/tools/ANATOMY.md - src/lingtai/tools/vision/__init__.py - - src/lingtai/services/vision/ANATOMY.md + - src/lingtai/tools/vision/CONTRACT.md - src/lingtai/tools/vision/glossary-en.md - src/lingtai/tools/vision/glossary-zh.md - src/lingtai/tools/vision/glossary-wen.md + - src/lingtai/tools/vision/manual/SKILL.md + - src/lingtai/services/vision/ANATOMY.md maintenance: | - Keep related_files as repo-relative paths to real files. Include neighboring - ANATOMY.md files so the anatomy graph stays connected rather than isolated; - anatomy links must be bidirectional. If you create a new ANATOMY.md, copy this - maintenance field. If you notice drift between this anatomy and the code, - report it. See lingtai-dev-guide for details. + Keep related_files as repo-relative paths to real files and keep anatomy links + reciprocal. Update citations with structural code changes and run the document + validators after edits. --- # src/lingtai/tools/vision/ -Vision capability — image understanding via pluggable VisionService backends. - -> **Maintenance:** see the `lingtai-kernel-anatomy` skill. **Coding agents** update this file in the same commit as code changes. **LingTai agents** report drift as issues. +The `vision` tool registers direct current-preset image analysis plus a +provider-neutral manual route when direct setup is unavailable. ## Components -| File | LOC | Role | -|---|---|---| -| `__init__.py` | 153 | `VisionManager`, `setup()`, provider registry, tool schema | - -**Key symbols:** -- `PROVIDERS` (L27-31) — supported providers: `minimax`, `zhipu`, `mimo`, `gemini`, `anthropic`, `openai`, `codex`. No static default; OpenAI-compat inherit fallback lives in `setup()`, not the registry. -- `VisionManager` (L53) — handles tool calls; resolves relative image paths via `agent._working_dir` (L73). -- `setup()` (L90) — entry point called by `capabilities.setup_capability()`. Creates `VisionManager`, registers `"vision"` tool on agent (L134). +- `__init__.py:34-41` — exact same-provider alias check; only GLM/Zhipu and codex-pool spelling pairs share current identity. +- `__init__.py:44-52` — exact advertised provider registry; the local pseudo-provider remains explicit opt-in and intentionally excluded. +- `__init__.py:58-76` — compatible tool schema; neither action requires an image path at schema level. +- `__init__.py:80-128` — `VisionManager`; `manual` reads bundled guidance without a backend, while `analyze` validates and reads the image. +- `__init__.py:131-379` — `setup`; resolves only the same current model/endpoint/credential/headers/wire, creates supported services, fails closed to manual guidance when identity is incomplete, and always registers the tool. ## Connections -- **→ `lingtai.i18n.t`** (L21) — i18n for tool description and schema strings. -- **→ `lingtai.services.vision.VisionService`** (L22) — abstract service interface + `create_vision_service()` factory. -- **→ `capabilities._media_host.resolve_media_host`** (L120) — injected for `minimax` provider. -- **→ `capabilities._zhipu_mode.resolve_z_ai_mode`** (L123) — injected for `zhipu` provider. -- **→ `lingtai.kernel.base_agent.BaseAgent`** — type-only (L25). -- **← `capabilities.__init__`** — registered as `".vision"` in `_BUILTIN`. +- Setup lazily reaches `lingtai.services.vision` and the Codex pool selector. +- Direct compatible aliases (`openrouter`, `deepseek`, `zhipu`, `glm`, `grok`, + `qwen`, `kimi`, `custom`) use current OpenAI/Anthropic-compatible identity. +- MiniMax uses Anthropic; Codex aliases use the Codex service; Claude Code and + unresolved/unsupported routes remain manual-only. No MCP fallback is used. ## Composition -Single file. No internal state — `VisionManager` instances hold agent + service refs. +`VisionManager` owns the agent, optional service, and safe manual reason. The +capability is registered by the built-in capability loader and registers one +`vision` tool with the schema and glossary package. ## State -- `VisionManager._agent` / `_vision_service` (L61-62) — per-agent instance state. Stateless tool handler otherwise. -- `PROVIDERS` dict is module-level constant. +Only the in-memory manager/service references persist. Manual content is bundled +with the package; analyses are not persisted. ## Notes -- OpenAI-compat fallback: if the agent's provider isn't in `PROVIDERS` but the main LLM's `_provider_defaults["api_compat"] == "openai"`, vision routes through `OpenAIVisionService` using the LLM's own `base_url`/`model`/`api_key`. Lets `custom`/`openrouter`/`deepseek`/`kimi` users opt into vision via `vision: {"provider": "inherit"}` in their preset. Succeeds only if the relay+model actually support OpenAI-style `image_url` content blocks; otherwise the runtime call surfaces the relay's error. -- Graceful skip: if the agent's provider isn't in `PROVIDERS` AND the LLM is not OpenAI-compatible, setup returns `None` silently. Agent logs `capability_skipped`. -- Codex is exposed through `PROVIDERS` and flows to `create_vision_service("codex", api_key=None)`; the service uses ChatGPT OAuth rather than an API key. -- Provider-specific kwarg injection is opt-in per provider — prevents `TypeError` from passing unsupported kwargs to heterogeneous service constructors. -- Local mlx-vlm provider exists in `services/vision/local.py` but is intentionally hidden from `PROVIDERS` (see docstring L11-14). +Setup failures retain provider plus exception type, never exception text. Direct +request failures likewise expose only the exception type and a manual pointer. +Active MiMo Responses/other unsupported wires are manual-only; supported Chat +Completions does not receive unsupported headers or wire kwargs. diff --git a/src/lingtai/tools/vision/CONTRACT.md b/src/lingtai/tools/vision/CONTRACT.md index 34bbfb262..3619e7a9a 100644 --- a/src/lingtai/tools/vision/CONTRACT.md +++ b/src/lingtai/tools/vision/CONTRACT.md @@ -5,134 +5,66 @@ contract_version: 1 related_files: - src/lingtai/tools/vision/__init__.py - src/lingtai/tools/vision/ANATOMY.md + - src/lingtai/tools/vision/manual/SKILL.md maintenance: | - Keep related_files as repo-relative paths to real files. If behavior and this - contract disagree, the code is the source of truth — fix the contract in the - same change and bump contract_version on breaking contract edits. + Keep this contract aligned with the vision tool and its tests. Bump the + version only for a repository-policy-required breaking contract change. --- - # Vision capability contract -`vision` analyzes a single image via a `VisionService`. It is a thin tool over a -provider service; all model/transport heterogeneity is resolved at setup time. -The implementation lives in `src/lingtai/tools/vision/`; the code is the source of -truth. - -## Routing Card - -**Use this when:** -- You are editing the vision tool surface, its `VisionService` wiring, or the - provider-resolution / api_compat fallback in `setup`. -- You are reviewing image-path handling (absolute vs working-dir-relative) or - the provider list advertised to first-run wizards. - -**Do not use this for:** -- Web/text search: use `web_search` (see `src/lingtai/tools/web_search/CONTRACT.md`). -- Code navigation only: read `src/lingtai/tools/vision/ANATOMY.md`. -- Adding a provider service implementation: that lives under - `src/lingtai/services/vision/`, imported lazily from here. - -**Fast paths:** tool schema -> §Tool surface; provider list -> §Scope; -lazy-import DAG rule -> §Cross-platform invariants. - -## Scope - -- Canonical tool name: `vision`. -- One tool, one call — no actions, no persistent state. -- Advertised providers (`PROVIDERS["providers"]`): `minimax`, `zhipu`, `mimo`, - `gemini`, `anthropic`, `openai`, `codex`. Default provider is `None` and there - is no agnostic inherit fallback (`fallback_on_inherit: None`). -- A local `mlx-vlm` provider (`provider="local"`) exists but is intentionally - **not** advertised in `PROVIDERS`; users opt in explicitly. - -**Non-goals:** the capability does not pre-validate that the resolved -model/relay can actually do vision — an incapable relay fails at runtime, not at -registration. It does not persist analyses or manage image storage. - -## Tool surface - -Schema requires `image_path`. - -| Inputs | Optional inputs | Success output | Error shapes | -|---|---|---|---| -| `image_path` (required) | `question` (default `"Describe this image."`; handler default `"Describe what you see in this image."`) | `{status: "ok", analysis: }` | `{status: "error", message}` — missing `image_path` (`Provide image_path`), file not found (`Image file not found: `), empty model response (`Vision analysis returned no response.`), or `Vision analysis failed: ` | - -`image_path` may be absolute or relative; relative paths are resolved against -the agent working directory before the file-existence check. - -## State & storage - -None. `vision` reads the given image file and returns the model's analysis. It -writes no files and keeps no per-call state under the agent working directory. - -## Cross-platform invariants - -DOCUMENT ONLY — do not change these assumptions and do not propose Windows work. - -- The provider service is imported **lazily inside `setup`** - (`from lingtai.services.vision import create_vision_service`, and the - api_compat-routed branches importing `OpenAIVisionService` / - `AnthropicVisionService`). This preserves the architectural DAG rule that the - `lingtai.tools → lingtai` import edge is only crossed inside setup/handlers, never at - module import. -- Provider-specific kwargs are injected per branch (e.g. MiniMax `api_host`, - Zhipu `z_ai_mode`) because vision services have heterogeneous constructor - signatures; `api_compat` / `base_url` are popped before forwarding to - dedicated services. -- Unknown providers route through the OpenAI- or Anthropic-compatible service - based on `api_compat`; if neither matches, the capability logs - `capability_skipped` and returns `CAPABILITY_UNAVAILABLE`. - -There are no subprocess/shell/PTY/binary-spawn assumptions in this tool. - -## Anchored claims - -| Claim | Source | Test | -|---|---|---| -| `setup` registers the `vision` tool with a provider or a service | `src/lingtai/tools/vision/__init__.py` | `tests/test_vision_capability.py::test_vision_added_by_setup`, `::test_vision_setup_with_provider_and_key`, `::test_vision_with_dedicated_service` | -| Setup raises when neither service nor provider is given | `src/lingtai/tools/vision/__init__.py` | `tests/test_vision_capability.py::test_vision_setup_requires_provider_or_service`, `::test_vision_setup_no_provider_raises` | -| Missing / relative `image_path` handling is correct | `src/lingtai/tools/vision/__init__.py` | `tests/test_vision_capability.py::test_vision_empty_image_path`, `::test_vision_missing_image`, `::test_vision_relative_path` | -| Empty model response is an error; service exceptions are caught | `src/lingtai/tools/vision/__init__.py` | `tests/test_vision_capability.py::test_vision_empty_response_is_error`, `::test_vision_service_error_handled` | -| Unsupported provider with an api_compat routes to the compat service | `src/lingtai/tools/vision/__init__.py` | `tests/test_vision_capability.py::test_vision_fallback_anthropic_compat_routes_to_anthropic_service`, `::test_vision_fallback_reads_api_compat_from_provider_bucket` | -| Unknown api_compat skips the capability with a diagnostic | `src/lingtai/tools/vision/__init__.py` | `tests/test_vision_capability.py::test_vision_fallback_unknown_api_compat_skips_with_diagnostic`, `::test_vision_setup_unsupported_provider_skips` | -| `api_key_env` overrides the raw key at setup | `src/lingtai/tools/vision/__init__.py` | `tests/test_vision_capability.py::test_vision_setup_resolves_api_key_env` | -| Dedicated provider services parse valid/invalid responses correctly | `src/lingtai/services/vision/` | `tests/test_vision_services.py::test_mimo_vision_returns_content_on_valid_response`, `::test_openai_vision_returns_content_on_valid_response`, `::test_create_vision_service_unknown_provider` | - -## Verification matrix - -| Invariant | Automated test | Manual check | Risk if broken | -|---|---|---|---| -| Tool registers only with a usable service/provider | `tests/test_vision_capability.py::test_vision_added_by_setup` | Boot with `capabilities={"vision": {"provider": "gemini", "api_key": "..."}}` | Agent silently lacks vision, or crashes at setup | -| Image path resolution + existence check | `tests/test_vision_capability.py::test_vision_relative_path` | Call with a relative path under the agent dir | False "not found" or reads outside intent | -| Provider-resolution fallback (api_compat) | `tests/test_vision_capability.py::test_vision_fallback_reads_api_compat_from_provider_bucket` | Configure an OpenRouter-style relay, confirm routing | Unknown providers hard-fail instead of routing | -| Lazy import preserves the lingtai.tools → lingtai DAG rule | import-time absence of `lingtai.services` at module load | `grep` for top-level `lingtai.services` imports (none) | Import cycle / layering violation | -| Errors are structured, never raised to the agent loop | `tests/test_vision_capability.py::test_vision_service_error_handled` | Point at an unreachable endpoint, confirm `status: error` | Tool-call crash instead of a recoverable error | - -Run before merging vision changes: - -```bash -python -m pytest tests/test_vision_capability.py tests/test_vision_services.py -q -``` - -## Schema and glossary ownership - -- **Canonical identifiers:** function names, JSON property names, action/enum - values, required fields, defaults, and bounds are canonical English literals. - The schema (`get_schema()`) and description (`get_description()`) are - language-independent; the optional `lang` argument is accepted for source - compatibility but ignored. -- **Provider wire:** provider adapters send the global `WIRE_TOOL_DESCRIPTION` - constant as the top-level tool description; `FunctionSchema.description` - holds the full canonical prose rendered into `## tools`. -- **Glossary resources:** this package owns `glossary-en.md`, `glossary-zh.md`, - and `glossary-wen.md`. Each has strict YAML frontmatter - (`kind: tool-glossary`, `schema_version: 1`, `tool_package: tools.`, - `language: `). English body is empty; zh/wen bodies contain concise - terminology mappings that quote immutable English identifiers and never offer - localized aliases. -- **Fallback:** exact normalized language lookup, then English, then no - appendix. Fail-closed for localized text; fail-open for tool availability. -- **Update triggers:** changing a function name, action/enum value, property - name, or user-visible concept requires reviewing all three glossary files in - the same PR. -- **Validation:** `python -m lingtai.tools.glossary_validator --check`. +`vision` analyzes one image through the active preset's current compatible route. +If direct setup is absent, unsupported, or fails, setup still registers the tool +and preserves a read-only `action="manual"` route. It never changes provider or +automatically invokes MCP. + +## Scope and registry + +The schema has optional `image_path`, `question`, and `action`; `action` is +`analyze` by default or `manual`. Manual works without `image_path`; analyze +requires it and resolves relative paths against the agent working directory. + +`PROVIDERS["providers"]` is exactly: `gemini`, `anthropic`, `openai`, +`openrouter`, `custom`, `deepseek`, `minimax`, `mimo`, `glm`, `zhipu`, `grok`, +`qwen`, `kimi`, `codex`, `codex-pool`, `codex_pool`, `claude-code`, and +`claude_code`. The local mlx-vlm pseudo-provider remains available only through +explicit `add_capability(..., provider="local")` opt-in and is intentionally not +advertised to wizards/check-caps. Claude Code is manual-only; Codex aliases use +native Codex Responses; MiniMax uses the Anthropic route; compatible aliases use +the current OpenAI/Anthropic identity. + +## Current identity and wires + +Direct routes inherit identity only from the same current provider (including the +explicit GLM/Zhipu and codex-pool spelling pairs); a different provider must supply +its own model and credential. Missing identity fails closed to `manual` instead of +using a service default model, a default OAuth path, or an SDK environment key. +OpenAI preserves current default headers, endpoint, model, and `wire_api`. +A missing, blank/whitespace-only, or `auto` selector means automatic selection: +the current route uses Responses only when it explicitly prefers Responses and +has no custom base URL; otherwise it uses Chat Completions. Unknown nonblank or +non-string selectors remain manual-only. Responses sends `max_output_tokens`. +MiniMax→Anthropic preserves active headers. MiMo accepts only API key/model/base +URL/max tokens: blank/auto resolves to its current Chat Completions route, which +constructs without headers/wire kwargs, while an active unsupported wire remains +manual-only. + +## Tool behavior + +Success is `{status: "ok", analysis: text}`. Manual success is +`{status: "ok", action: "manual", manual: body}`; missing manual is degraded. +Missing image, empty response, setup failure, and request failure are structured +errors pointing to `vision(action="manual")`. Exception messages are never +returned; failures may include only the provider and exception type. + +## Invariants and tests + +- `setup` always registers the tool: `tests/test_vision_capability.py`. +- Endpoint identity is sanitized by `sanitize_endpoint` and drops userinfo, + query, fragment, malformed ports, and non-URLs: `tests/test_agent_preset_manifest.py`. +- Provider construction and exact OpenAI Responses shape are covered in + `tests/test_vision_capability.py`. +- Manual guidance is provider-neutral and kernel/TUI-independent in + `manual/SKILL.md`. + +Run `python -m pytest tests/test_vision_capability.py tests/test_vision_services.py -q` +and the glossary validator before merging. diff --git a/src/lingtai/tools/vision/__init__.py b/src/lingtai/tools/vision/__init__.py index dcaaf737d..6f8f7011d 100644 --- a/src/lingtai/tools/vision/__init__.py +++ b/src/lingtai/tools/vision/__init__.py @@ -7,32 +7,70 @@ agent.add_capability("vision", vision_service=my_svc) agent.add_capability("vision", provider="anthropic", api_key="sk-...") -Note: a local mlx-vlm provider exists (``provider="local"``) and works -on Apple Silicon, but it is intentionally NOT exposed in ``PROVIDERS`` -below so that first-run wizards and check-caps don't advertise it yet. -Users who want it can opt in explicitly via ``add_capability`` with -``provider="local"``; see ``services/vision/local.py``. +The local mlx-vlm pseudo-provider remains available through explicit +``add_capability(..., provider="local")`` opt-in, but it is intentionally not +advertised in ``PROVIDERS`` or first-run/check-caps surfaces yet. """ from __future__ import annotations from pathlib import Path +from importlib import resources from typing import TYPE_CHECKING, Any -from ..registry import CAPABILITY_UNAVAILABLE - if TYPE_CHECKING: from lingtai.kernel.base_agent import BaseAgent from lingtai.services.vision import VisionService + +def _setup_failure(provider: str, exc: BaseException) -> str: + """Build explicit manual guidance without exposing exception contents.""" + return ( + f"Direct vision setup failed for provider {provider!r} " + f"({type(exc).__name__}); use vision(action='manual')." + ) + + +def _same_provider_identity(requested: str, active: str) -> bool: + """Return whether two provider names identify the same current route.""" + if requested == active: + return True + return {requested, active} <= {"glm", "zhipu"} or { + requested, + active, + } <= {"codex-pool", "codex_pool"} + + +def _effective_openai_wire( + wire_api: str | None, + *, + use_responses_api: bool, + base_url: str | None, +) -> str | None: + """Resolve a supported canonical wire; reject unknown protocols.""" + normalized = wire_api.strip().lower() if isinstance(wire_api, str) else wire_api + if isinstance(normalized, str): + if normalized in {"chat_completions", "responses"}: + return normalized + if normalized in {"", "auto"}: + return "responses" if use_responses_api and not base_url else "chat_completions" + elif normalized is None: + return "responses" if use_responses_api and not base_url else "chat_completions" + return None + + PROVIDERS = { - "providers": ["minimax", "zhipu", "mimo", "gemini", "anthropic", "openai", "codex"], + "providers": [ + "gemini", "anthropic", "openai", "openrouter", "custom", "deepseek", + "minimax", "mimo", "glm", "zhipu", "grok", "qwen", "kimi", + "codex", "codex-pool", "codex_pool", "claude-code", "claude_code", + ], "default": None, "fallback_on_inherit": None, # no agnostic fallback for vision } def get_description(lang: str = "en") -> str: - return "Analyze an image using the LLM's vision capabilities. Supports JPEG, PNG, and WebP. Ask any question about the image — describe contents, read text, interpret charts, identify objects, assess style or mood. Combine with draw to generate then analyze images." + return "Analyze an image with the active preset when directly supported. Use action='manual' for read-only guidance when unsupported or after a direct failure. No provider or MCP fallback is automatic." def get_schema(lang: str = "en") -> dict: @@ -45,8 +83,14 @@ def get_schema(lang: str = "en") -> dict: "description": 'Question about the image', "default": "Describe this image.", }, + "action": { + "type": "string", + "enum": ["analyze", "manual"], + "default": "analyze", + "description": "manual returns bundled read-only vision guidance; analyze performs the direct request.", + }, }, - "required": ["image_path"], + "required": [], } @@ -57,12 +101,18 @@ class VisionManager: def __init__( self, agent: "BaseAgent", - vision_service: VisionService, + vision_service: VisionService | None, + manual_reason: str = "", ) -> None: self._agent = agent self._vision_service = vision_service + self._manual_reason = manual_reason def handle(self, args: dict) -> dict: + if args.get("action", "analyze") == "manual": + return self.manual() + if self._vision_service is None: + return {"status": "error", "message": self._manual_reason or "Direct vision is unavailable; call vision(action='manual')."} image_path = args.get("image_path", "") question = args.get("question", "Describe what you see in this image.") @@ -85,7 +135,15 @@ def handle(self, args: dict) -> dict: } return {"status": "ok", "analysis": analysis} except Exception as e: - return {"status": "error", "message": f"Vision analysis failed: {e}"} + return {"status": "error", "message": f"Vision analysis failed ({type(e).__name__}). Call vision(action='manual') for the explicit manual route."} + + def manual(self) -> dict: + """Return only bundled guidance; never inspect config or invoke a backend.""" + try: + body = resources.files(__package__).joinpath("manual/SKILL.md").read_text(encoding="utf-8") + except (FileNotFoundError, ModuleNotFoundError, AttributeError): + return {"status": "degraded", "action": "manual", "manual": "", "error": "vision manual missing"} + return {"status": "ok", "action": "manual", "manual": body} def setup( @@ -98,14 +156,41 @@ def setup( ) -> VisionManager: """Set up the vision capability on an agent. - Requires either ``vision_service`` or ``provider`` + ``api_key``. - Raises ``ValueError`` if neither is provided. + Requires either ``vision_service`` or ``provider`` + ``api_key`` for a + direct route. Without one, the tool is still registered for manual guidance. """ + manual_reason = "" if vision_service is None and provider is not None: if api_key_env: from lingtai.kernel.config_resolve import resolve_env api_key = resolve_env(api_key, api_key_env) - if provider not in PROVIDERS["providers"]: + provider_key = provider.lower() + active_service = getattr(agent, "service", None) + active_provider = getattr(active_service, "provider", "") + active_provider_key = active_provider.lower() if isinstance(active_provider, str) else "" + same_provider = _same_provider_identity(provider_key, active_provider_key) + active_model = getattr(active_service, "_model", None) if same_provider else None + active_base_url = getattr(active_service, "_base_url", None) if same_provider else None + active_api_key = getattr(active_service, "api_key", None) if same_provider else None + if provider_key == "local": + # Local vision is an explicit pseudo-provider: keep it out of + # PROVIDERS/check-caps, but preserve the documented opt-in route. + # Its constructor accepts only model/max_tokens and needs no key. + local_kwargs = { + key: kwargs[key] + for key in ("model", "max_tokens") + if key in kwargs and kwargs[key] is not None + } + from lingtai.services.vision import create_vision_service + try: + vision_service = create_vision_service( + "local", + api_key=None, + **local_kwargs, + ) + except Exception as exc: + manual_reason = _setup_failure(provider, exc) + elif provider_key not in PROVIDERS["providers"]: # No dedicated VisionService for this provider (custom relay, # OpenRouter, an anthropic-compat local proxy, ...). Route vision # through the OpenAI- or Anthropic-compatible service, picking the @@ -119,81 +204,202 @@ def setup( # service._base_url / service._model. # If the relay or model can't actually do vision, the call fails at # runtime — capability registration never pre-checks. + bucket = {} api_compat = (kwargs.get("api_compat") or "").lower() if not api_compat: - defaults = getattr(getattr(agent, "service", None), "_provider_defaults", None) + defaults = getattr(active_service, "_provider_defaults", None) if same_provider else None if isinstance(defaults, dict): # _provider_defaults is dict[provider_name, defaults_dict]; - # read the bucket for *this* provider, not the outer dict. - bucket = defaults.get((provider or "").lower()) + # read only the active provider's bucket, never another + # provider's credential/transport configuration. + bucket = defaults.get(active_provider_key) if isinstance(bucket, dict): api_compat = (bucket.get("api_compat") or "").lower() cap_model = kwargs.get("model") cap_base_url = kwargs.get("base_url") cap_max_tokens = kwargs.get("max_tokens") - llm_base_url = cap_base_url or getattr(agent.service, "_base_url", None) - llm_model = cap_model or getattr(agent.service, "_model", None) + bucket = bucket if isinstance(bucket, dict) else {} + llm_base_url = cap_base_url or active_base_url or bucket.get("base_url") + llm_model = cap_model or active_model or bucket.get("model") + api_key = api_key or active_api_key + headers = kwargs.get("default_headers") or bucket.get("default_headers") + wire_api = _effective_openai_wire( + kwargs.get("wire_api") or bucket.get("wire_api"), + use_responses_api=bucket.get("use_responses_api") is True, + base_url=llm_base_url, + ) if api_compat == "openai": from lingtai.services.vision.openai import OpenAIVisionService svc_kwargs: dict = { "api_key": api_key, - "model": llm_model or "gpt-4o", + "model": llm_model, "base_url": llm_base_url, } + if headers: + svc_kwargs["default_headers"] = headers + if wire_api and wire_api != "auto": + svc_kwargs["wire_api"] = wire_api if cap_max_tokens is not None: svc_kwargs["max_tokens"] = cap_max_tokens - vision_service = OpenAIVisionService(**svc_kwargs) + if wire_api is None: + manual_reason = "The active OpenAI-compatible wire is not implemented by the direct vision service; use vision(action='manual')." + elif not llm_model: + manual_reason = f"Provider {provider!r} has no resolved current model for direct vision; use vision(action='manual')." + elif not api_key: + manual_reason = f"Provider {provider!r} has no resolved current credential for direct vision; use vision(action='manual')." + else: + try: + vision_service = OpenAIVisionService(**svc_kwargs) + except Exception as exc: + manual_reason = _setup_failure(provider, exc) elif api_compat == "anthropic": from lingtai.services.vision.anthropic import AnthropicVisionService svc_kwargs = { "api_key": api_key, - "model": llm_model or "claude-sonnet-4-20250514", + "model": llm_model, "base_url": llm_base_url, } + if headers: + svc_kwargs["default_headers"] = headers if cap_max_tokens is not None: svc_kwargs["max_tokens"] = cap_max_tokens - vision_service = AnthropicVisionService(**svc_kwargs) + if not llm_model: + manual_reason = f"Provider {provider!r} has no resolved current model for direct vision; use vision(action='manual')." + elif not api_key: + manual_reason = f"Provider {provider!r} has no resolved current credential for direct vision; use vision(action='manual')." + else: + try: + vision_service = AnthropicVisionService(**svc_kwargs) + except Exception as exc: + manual_reason = _setup_failure(provider, exc) else: - agent._log( - "capability_skipped", - capability="vision", - requested_provider=provider, - reason=( - f"no vision support for provider {provider!r} " - f"(api_compat={api_compat!r})" - ), - ) - return CAPABILITY_UNAVAILABLE + manual_reason = f"No direct vision route is supported for provider {provider!r}; use vision(action='manual')." else: - # Provider-specific kwarg injection. Each branch is opt-in because - # vision services have heterogeneous constructor signatures — - # passing api_host to a service that doesn't accept it raises - # TypeError at construction (silently swallowed by the agent's - # capability-setup try/except, leaving the agent without vision). - if provider == "minimax" and "api_host" not in kwargs: - from .._media_host import resolve_media_host - kwargs["api_host"] = resolve_media_host(agent) - if provider == "zhipu" and "z_ai_mode" not in kwargs: - from .._zhipu_mode import resolve_z_ai_mode - kwargs["z_ai_mode"] = resolve_z_ai_mode(agent) - # Dedicated vision services do not consume the LLM adapter's - # transport selector. ``expand_inherit`` copies it for capability - # routing, but forwarding it into service constructors breaks - # providers such as MiniMax that only accept provider-native args. - kwargs.pop("api_compat", None) - kwargs.pop("base_url", None) - # Lazy import: the provider service lives in ``lingtai.services`` and - # the ``lingtai.tools → lingtai`` edge is allowed only inside setup/handlers. - from lingtai.services.vision import create_vision_service - vision_service = create_vision_service(provider, api_key=api_key, **kwargs) + if provider_key in {"codex", "codex-pool", "codex_pool"}: + # Codex vision is a standalone Responses request. It may share + # the active Codex family's model and endpoint, but never + # inherits those from an unrelated main provider. + if same_provider: + if active_model: + kwargs.setdefault("model", active_model) + if active_base_url: + kwargs.setdefault("base_url", active_base_url) + codex_base_url = kwargs.get("base_url") + + defaults = getattr(active_service, "_provider_defaults", None) if same_provider else None + bucket = defaults.get(active_provider_key) if isinstance(defaults, dict) else None + if not isinstance(bucket, dict): + bucket = {} + if not kwargs.get("model"): + manual_reason = f"Provider {provider!r} has no resolved current model for direct vision; use vision(action='manual')." + elif provider_key == "codex": + token_path = kwargs.pop("token_path", None) or bucket.get("codex_auth_path") + if token_path: + kwargs["token_path"] = token_path + else: + manual_reason = "Codex vision has no explicit current OAuth identity; use vision(action='manual')." + else: + # The pool selector is the single owner of deterministic + # account choice. It reads only the current pool's + # non-secret file; an unrelated active provider is ignored. + if same_provider: + from lingtai.auth.codex_pool import select_codex_pool_auth + selection = select_codex_pool_auth( + bucket, + model=kwargs.get("model"), + ) + if selection: + kwargs["token_path"] = selection["auth_path"] + if not kwargs.get("token_path"): + manual_reason = "Codex pool vision has no selected current OAuth identity; use vision(action='manual')." + kwargs.pop("api_compat", None) + kwargs.pop("base_url", None) + if codex_base_url: + kwargs["base_url"] = codex_base_url + if not manual_reason: + from lingtai.services.vision import create_vision_service + try: + vision_service = create_vision_service("codex", api_key=None, **kwargs) + except Exception as exc: + manual_reason = _setup_failure(provider, exc) + else: + service_provider = provider_key + defaults = getattr(active_service, "_provider_defaults", {}) if same_provider else {} + bucket = defaults.get(active_provider_key, {}) if isinstance(defaults, dict) else {} + active_base_url = active_base_url or (bucket.get("base_url") if isinstance(bucket, dict) else None) + active_headers = bucket.get("default_headers") if isinstance(bucket, dict) else None + active_compat = kwargs.get("api_compat") or (bucket.get("api_compat") if isinstance(bucket, dict) else "") or "" + wire_api = _effective_openai_wire( + kwargs.get("wire_api") or (bucket.get("wire_api") if isinstance(bucket, dict) else None), + use_responses_api=isinstance(bucket, dict) and bucket.get("use_responses_api") is True, + base_url=kwargs.get("base_url") or active_base_url, + ) + if service_provider in {"openrouter", "deepseek", "zhipu", "glm", "grok", "qwen", "kimi"}: + service_provider = "anthropic" if active_compat.lower() == "anthropic" else "openai" + elif service_provider == "custom": + service_provider = "anthropic" if active_compat.lower() == "anthropic" else "openai" + + # Provider-specific kwarg injection. Each branch is opt-in because + # vision services have heterogeneous constructor signatures. + if service_provider == "minimax": + service_provider = "anthropic" + if service_provider in {"openai", "anthropic", "gemini", "mimo"}: + if same_provider and active_model: + kwargs.setdefault("model", active_model) + if ( + service_provider in {"openai", "anthropic"} + and same_provider + and active_base_url + ): + kwargs.setdefault("base_url", active_base_url) + if service_provider == "mimo" and same_provider and active_base_url: + kwargs.setdefault("base_url", active_base_url) + if service_provider in {"openai", "mimo"} and wire_api is None: + manual_reason = "The active OpenAI-compatible wire is not implemented by the direct vision service; use vision(action='manual')." + elif service_provider == "mimo" and wire_api != "chat_completions": + manual_reason = "The active MiMo wire is not implemented by the direct vision service; use vision(action='manual')." + if service_provider in {"openai", "mimo"} and active_compat == "anthropic": + manual_reason = "The active preset uses an Anthropic wire that this vision route cannot safely adapt; use vision(action='manual')." + vision_service = None + if service_provider == "anthropic" and active_headers: + kwargs.setdefault("default_headers", active_headers) + elif service_provider == "openai": + if active_headers: + kwargs.setdefault("default_headers", active_headers) + if wire_api not in (None, "auto"): + kwargs.setdefault("wire_api", wire_api) + elif service_provider == "mimo": + # MiMo's standalone constructor intentionally accepts only + # api_key/model/base_url/max_tokens. Its current direct + # route is Chat Completions; other wires stay manual-only. + kwargs.pop("default_headers", None) + kwargs.pop("wire_api", None) + resolved_api_key = api_key or active_api_key + if service_provider not in {"codex", "local"} and not kwargs.get("model"): + manual_reason = f"Provider {provider!r} has no resolved current model for direct vision; use vision(action='manual')." + elif service_provider not in {"codex", "local"} and not resolved_api_key: + manual_reason = f"Provider {provider!r} has no resolved current credential for direct vision; use vision(action='manual')." + # Dedicated vision services do not consume the LLM adapter's + # transport selector. + kwargs.pop("api_compat", None) + if service_provider not in {"openai", "anthropic", "mimo"}: + kwargs.pop("base_url", None) + # Lazy import: the provider service lives in ``lingtai.services``. + from lingtai.services.vision import create_vision_service + if vision_service is None and not manual_reason: + try: + vision_service = create_vision_service( + service_provider, + api_key=resolved_api_key, + **kwargs, + ) + except Exception as exc: + manual_reason = _setup_failure(provider, exc) elif vision_service is None: - raise ValueError( - "vision capability requires 'vision_service' or 'provider' + 'api_key'. " - "Example: capabilities={'vision': {'provider': 'gemini', 'api_key': '...'}}" - ) + manual_reason = "No direct vision provider was configured; use vision(action='manual')." - mgr = VisionManager(agent, vision_service=vision_service) + mgr = VisionManager(agent, vision_service=vision_service, manual_reason=manual_reason) agent.add_tool("vision", schema=get_schema(), handler=mgr.handle, description=get_description(), glossary_package=__package__) return mgr diff --git a/src/lingtai/tools/vision/glossary-wen.md b/src/lingtai/tools/vision/glossary-wen.md index 1bf683003..feb407d83 100644 --- a/src/lingtai/tools/vision/glossary-wen.md +++ b/src/lingtai/tools/vision/glossary-wen.md @@ -17,3 +17,5 @@ maintenance: | - `vision`：观象之器——以 LLM 之视觉能力析图。支 JPEG、PNG 与 WebP。可对图发任何问——述其内容、识其文字、解其图表、辨其物象、评其风格与气韵。结合绘相可先生图再析之。 - `image_path`：图像之路径 - `question`：关于图像之问 +- `action`：`analyze` 直析，`manual` 惟返只读指引 +- `manual`：使 agent 察当前 preset 身份，于自身 skill catalog 寻相应手册；不自动召 MCP diff --git a/src/lingtai/tools/vision/glossary-zh.md b/src/lingtai/tools/vision/glossary-zh.md index 6cfa4a6b7..5ee210df4 100644 --- a/src/lingtai/tools/vision/glossary-zh.md +++ b/src/lingtai/tools/vision/glossary-zh.md @@ -17,3 +17,5 @@ maintenance: | - `vision`：使用 LLM 的视觉能力分析图像。支持 JPEG、PNG 和 WebP。可以对图像提出任何问题——描述内容、识别文字、解读图表、识别物体、评估风格或氛围。结合 draw 可以先生成图像再分析。 - `image_path`：图像文件路径 - `question`：关于图像的问题 +- `action`：`analyze` 直接分析，`manual` 仅返回只读指引 +- `manual`：引导 agent 查看当前 preset 身份并在自身 skill catalog 中查找匹配手册；不自动调用 MCP diff --git a/src/lingtai/tools/vision/manual/SKILL.md b/src/lingtai/tools/vision/manual/SKILL.md new file mode 100644 index 000000000..0c6095bb9 --- /dev/null +++ b/src/lingtai/tools/vision/manual/SKILL.md @@ -0,0 +1,42 @@ +--- +name: vision-manual +description: > + Use this manual when the vision capability has no usable provider route or + reports a direct setup/request failure and needs safe, provider-neutral + troubleshooting guidance. +related_files: + - src/lingtai/tools/vision/__init__.py + - src/lingtai/tools/vision/ANATOMY.md + - src/lingtai/tools/vision/CONTRACT.md +maintenance: | + Keep this manual provider-neutral and read-only. It must not import, name, or + link to a TUI package, credential, endpoint secret, or automatic MCP action. +--- +# Vision manual + +This is the provider-neutral fallback for `vision`. It contains guidance only; +it does not discover, install, start, or invoke a backend. + +## Direct route + +When `vision` reports a direct setup or request failure, inspect the identity +already shown in the prompt: the current provider, model, and sanitized +endpoint. Do not substitute another provider, model, credential, endpoint, or +wire protocol. Retry only after the operator has corrected the active preset. + +## Find the current preset's method + +Use the `skills` capability's catalog to search your own installed skills for a +manual matching that provider/model or preset. Read the matching manual before +trying its documented method or official-page pointer. If no matching manual is +present, report that no discoverable vision method is available. + +An optional MCP or other skill may be described by that preset manual, but it +is always an explicit operator/agent action. This manual never auto-loads or +auto-invokes MCP. + +## Safety + +Never request or print API keys, OAuth tokens, environment values, headers, or +full unsanitized URLs. Missing provider, model, or endpoint fields are simply +unknown; do not fill them with guesses. diff --git a/tests/test_agent_capabilities.py b/tests/test_agent_capabilities.py index 884a02087..be9d356d9 100644 --- a/tests/test_agent_capabilities.py +++ b/tests/test_agent_capabilities.py @@ -2,7 +2,6 @@ from __future__ import annotations import pytest -from unittest.mock import MagicMock from lingtai.agent import Agent from lingtai.services.vision import VisionService from lingtai.services.websearch import SearchService, SearchResult @@ -45,8 +44,8 @@ def test_agent_no_capabilities_boots_core_floor(tmp_path): agent.stop(timeout=1.0) -def test_agent_unsupported_vision_skip_is_not_registered(tmp_path): - """A skipped capability must not appear registered or callable.""" +def test_agent_unsupported_vision_registers_manual_route(tmp_path): + """Unsupported vision remains registered for explicit manual guidance.""" agent = Agent( service=make_mock_service(), agent_name="test", @@ -57,10 +56,10 @@ def test_agent_unsupported_vision_skip_is_not_registered(tmp_path): manifest_registered = { name for name, _ in agent._build_manifest().get("capabilities", []) } - assert agent.has_capability("vision") is False - assert agent.get_capability("vision") is None - assert "vision" not in agent._tool_handlers - assert "vision" not in manifest_registered + assert agent.has_capability("vision") is True + assert agent.get_capability("vision") is not None + assert "vision" in agent._tool_handlers + assert "vision" in manifest_registered finally: agent.stop(timeout=1.0) @@ -170,17 +169,13 @@ def test_agent_seal_after_start(tmp_path): def test_vision_requires_provider(tmp_path): - """Vision capability is skipped when no provider or service is given. - - setup() raises ValueError, but the agent catches it (capability_skipped) - and simply doesn't register the tool. - """ + """No provider still registers vision's manual route.""" agent = Agent( service=make_mock_service(), agent_name="test", working_dir=tmp_path / "test", capabilities=["vision"], ) - assert agent.get_capability("vision") is None - assert "vision" not in {s.name for s in agent._tool_schemas} + assert agent.get_capability("vision") is not None + assert "vision" in {s.name for s in agent._tool_schemas} agent.stop(timeout=1.0) diff --git a/tests/test_agent_preset_manifest.py b/tests/test_agent_preset_manifest.py index 2cd010edb..78704a6a6 100644 --- a/tests/test_agent_preset_manifest.py +++ b/tests/test_agent_preset_manifest.py @@ -16,7 +16,7 @@ from lingtai.agent import Agent from lingtai.kernel.base_agent import BaseAgent, _build_identity_section -from lingtai.kernel.base_agent.identity import _build_manifest, _safe_llm_from_service +from lingtai.kernel.base_agent.identity import _build_manifest, _safe_llm_from_service, sanitize_endpoint from tests._workdir_lease_helpers import make_test_lease from tests._snapshot_helpers import make_test_snapshot_port, make_test_source_revision_port from tests._lifecycle_clock_helpers import make_test_lifecycle_clock @@ -128,6 +128,22 @@ def test_safe_llm_from_service_unit(): } +def test_sanitize_endpoint_removes_userinfo_query_and_fragment(): + assert sanitize_endpoint("https://user:secret@example.test/v1?key=secret#frag") == "https://example.test/v1" + + +def test_sanitize_endpoint_preserves_ipv6_and_port_without_secrets(): + assert sanitize_endpoint("https://user:secret@[::1]:8443/v1?token=x#frag") == "https://[::1]:8443/v1" + + +def test_sanitize_endpoint_rejects_malformed_port_with_credentials(): + assert sanitize_endpoint("https://user:secret@example.test:not-a-port/v1") == "" + + +def test_sanitize_endpoint_rejects_non_url_secret_like_input(): + assert sanitize_endpoint("api_key=super-secret-value") == "" + + def test_safe_llm_from_service_uses_provider_default_base_url(): agent = MagicMock() svc = _mock_service("custom", "model-x", None) diff --git a/tests/test_tools_package_data.py b/tests/test_tools_package_data.py index 8b9f2aecc..5799274f9 100644 --- a/tests/test_tools_package_data.py +++ b/tests/test_tools_package_data.py @@ -163,6 +163,10 @@ def test_wheel_ships_every_tool_contract(wheel_entries: set[str]): assert not missing, "tool contracts missing from wheel: %r" % missing +def test_wheel_ships_vision_manual(wheel_entries: set[str]): + assert "lingtai/tools/vision/manual/SKILL.md" in wheel_entries + + def test_wheel_ships_exact_expected_tool_contracts(wheel_entries: set[str]): # Keep the manifest closed: the 18 top-level tool contracts plus the one # intentional daemon component contract. No other nested/manual contract diff --git a/tests/test_validate_skill.py b/tests/test_validate_skill.py index f3c1ec589..6fa8dcac9 100644 --- a/tests/test_validate_skill.py +++ b/tests/test_validate_skill.py @@ -9,12 +9,10 @@ import sys from pathlib import Path -import pytest - # The validator lives outside the package tree; import it directly. _SCRIPTS = Path(__file__).resolve().parents[1] / "src" / "lingtai" / "tools" / "skills" / "manual" / "scripts" sys.path.insert(0, str(_SCRIPTS)) -from validate import _parse_frontmatter, validate_frontmatter # noqa: E402 +from validate import _parse_frontmatter, validate_frontmatter, validate_skill # noqa: E402 # --------------------------------------------------------------------------- @@ -156,6 +154,12 @@ def test_actual_missing_description_errors(self, tmp_path): assert not passed assert any("Missing 'description'" in m for m in msgs) + def test_vision_manual_passes_repository_validator(self): + """The shipped vision manual must pass the repository validator directly.""" + root = Path(__file__).resolve().parents[1] + manual = root / "src" / "lingtai" / "tools" / "vision" / "manual" + assert validate_skill(str(manual)) + def test_name_as_number_does_not_crash(self, tmp_path): """Non-string YAML scalar names should be handled gracefully.""" skill_dir = tmp_path / "skills" / "num-name" diff --git a/tests/test_vision_capability.py b/tests/test_vision_capability.py index ead6d9737..506ce9d7e 100644 --- a/tests/test_vision_capability.py +++ b/tests/test_vision_capability.py @@ -1,14 +1,15 @@ """Tests for vision capability and VisionService.""" from __future__ import annotations +import os +import subprocess import sys from types import SimpleNamespace from unittest.mock import MagicMock, patch import pytest -from lingtai.tools.registry import CAPABILITY_UNAVAILABLE -from lingtai.tools.vision import VisionManager, setup +from lingtai.tools.vision import PROVIDERS, VisionManager, setup from lingtai.services.vision import VisionService, create_vision_service @@ -16,6 +17,7 @@ def make_mock_service(): svc = MagicMock() svc.provider = "gemini" svc.model = "gemini-test" + svc.api_key = None svc._key_resolver = MagicMock(return_value="fake-key") return svc @@ -29,6 +31,24 @@ def make_mock_agent(tmp_path, svc=None): return agent +def make_provider_agent( + tmp_path, + *, + provider: str, + model: str | None, + base_url: str | None, + defaults: dict | None = None, +): + svc = MagicMock() + svc.provider = provider + svc._model = model + svc._base_url = base_url + svc._provider_defaults = defaults if defaults is not None else {provider: {}} + svc.api_key = None + svc._key_resolver = MagicMock(return_value="fake-key") + return make_mock_agent(tmp_path, svc=svc) + + def test_vision_added_by_setup(tmp_path): """setup() should register the vision tool on the agent.""" mock_svc = MagicMock(spec=VisionService) @@ -89,7 +109,143 @@ def test_vision_service_error_handled(tmp_path): img_path.write_bytes(b"\x89PNG fake") result = mgr.handle({"image_path": str(img_path)}) assert result["status"] == "error" - assert "API down" in result["message"] + assert "API down" not in result["message"] + assert "RuntimeError" in result["message"] + + +def test_vision_service_error_does_not_echo_secret_or_url(tmp_path): + mock_vision_svc = MagicMock(spec=VisionService) + mock_vision_svc.analyze_image.side_effect = RuntimeError( + "token=secret https://user:pw@example.test/v1" + ) + agent = make_mock_agent(tmp_path) + mgr = VisionManager(agent, vision_service=mock_vision_svc) + img_path = tmp_path / "test.png" + img_path.write_bytes(b"fake") + result = mgr.handle({"image_path": str(img_path)}) + assert result["status"] == "error" + assert "secret" not in result["message"] + assert "example.test" not in result["message"] + + +@pytest.mark.parametrize( + "provider", + ["openrouter", "deepseek", "zhipu", "glm", "grok", "qwen", "kimi", "custom"], +) +def test_compatible_aliases_build_current_openai_route(tmp_path, provider): + headers = {"X-Preset": "active"} + with patch("lingtai.services.vision.create_vision_service") as mock_factory: + mock_factory.return_value = MagicMock(spec=VisionService) + agent = make_provider_agent( + tmp_path, + provider=provider, + model="vision-current", + base_url="https://relay.example/v1", + defaults={provider: {"api_compat": "openai", "default_headers": headers, "wire_api": "chat_completions"}}, + ) + setup(agent, provider=provider, api_key="sk-test") + assert mock_factory.call_args.args == ("openai",) + assert mock_factory.call_args.kwargs == { + "api_key": "sk-test", + "model": "vision-current", + "base_url": "https://relay.example/v1", + "default_headers": headers, + "wire_api": "chat_completions", + } + + +@pytest.mark.parametrize("wire_api", ["auto", "", " \t "]) +@pytest.mark.parametrize( + ("base_url", "expected_wire"), + [ + (None, "responses"), + ("https://openai-compatible.example/v1", "chat_completions"), + ], +) +def test_openai_automatic_wire_values_preserve_active_effective_route( + tmp_path, wire_api, base_url, expected_wire +): + with patch("lingtai.services.vision.create_vision_service") as mock_factory: + mock_factory.return_value = MagicMock(spec=VisionService) + agent = make_provider_agent( + tmp_path, + provider="openai", + model="gpt-5.4", + base_url=base_url, + defaults={ + "openai": { + "api_compat": "openai", + "wire_api": wire_api, + "use_responses_api": True, + } + }, + ) + setup(agent, provider="openai", api_key="sk-test") + + assert mock_factory.call_args.args == ("openai",) + assert mock_factory.call_args.kwargs["wire_api"] == expected_wire + + +def test_openai_unknown_wire_remains_manual_without_factory_call(tmp_path): + with patch("lingtai.services.vision.create_vision_service") as mock_factory: + agent = make_provider_agent( + tmp_path, + provider="openai", + model="gpt-5.4", + base_url=None, + defaults={ + "openai": { + "api_compat": "openai", + "wire_api": "unproven_wire", + "use_responses_api": True, + } + }, + ) + mgr = setup(agent, provider="openai", api_key="sk-test") + + mock_factory.assert_not_called() + assert mgr._vision_service is None + result = mgr.handle({}) + assert result["status"] == "error" + assert "manual" in result["message"] + assert "unproven_wire" not in result["message"] + + +def test_generic_openai_compatible_unknown_wire_remains_manual(tmp_path): + with patch("lingtai.services.vision.openai.OpenAIVisionService") as mock_cls: + agent = make_provider_agent( + tmp_path, + provider="openai-compatible-relay", + model="vision-current", + base_url="https://relay.example/v1", + defaults={ + "openai-compatible-relay": { + "api_compat": "openai", + "wire_api": "unproven_wire", + } + }, + ) + mgr = setup( + agent, + provider="openai-compatible-relay", + api_key="sk-test", + ) + + mock_cls.assert_not_called() + assert mgr._vision_service is None + result = mgr.handle({}) + assert result["status"] == "error" + assert "manual" in result["message"] + assert "unproven_wire" not in result["message"] + + +def test_claude_code_remains_manual_only(tmp_path): + agent = make_provider_agent( + tmp_path, provider="claude-code", model="text-only", base_url="https://relay.example/v1" + ) + mgr = setup(agent, provider="claude-code", api_key="sk-test") + assert mgr._vision_service is None + assert mgr.manual()["status"] in {"ok", "degraded"} def test_vision_empty_response_is_error(tmp_path): @@ -111,13 +267,97 @@ def test_vision_setup_with_provider_and_key(tmp_path): mock_svc = MagicMock(spec=VisionService) mock_factory.return_value = mock_svc - agent = make_mock_agent(tmp_path) + agent = make_provider_agent( + tmp_path, + provider="anthropic", + model="claude-sonnet-4-20250514", + base_url=None, + ) mgr = setup(agent, provider="anthropic", api_key="sk-test") - mock_factory.assert_called_once_with("anthropic", api_key="sk-test") + mock_factory.assert_called_once_with( + "anthropic", + api_key="sk-test", + model="claude-sonnet-4-20250514", + ) assert isinstance(mgr, VisionManager) +def test_local_vision_is_hidden_but_explicitly_constructible(tmp_path): + """The local pseudo-provider stays out of discovery yet keeps its opt-in path.""" + assert "local" not in PROVIDERS["providers"] + + with patch("lingtai.services.vision.create_vision_service") as mock_factory: + mock_svc = MagicMock(spec=VisionService) + mock_factory.return_value = mock_svc + agent = make_mock_agent(tmp_path) + + mgr = setup( + agent, + provider="local", + model="mlx-community/local-test-model", + max_tokens=128, + api_compat="openai", + base_url="https://must-not-be-forwarded.example/v1", + ) + + mock_factory.assert_called_once_with( + "local", + api_key=None, + model="mlx-community/local-test-model", + max_tokens=128, + ) + assert mgr._vision_service is mock_svc + agent.add_tool.assert_called_once() + + +def test_minimax_vision_preserves_active_default_headers(tmp_path): + headers = {"X-Preset": "active"} + with patch("lingtai.services.vision.create_vision_service") as mock_factory: + mock_factory.return_value = MagicMock(spec=VisionService) + agent = make_provider_agent( + tmp_path, + provider="minimax", + model="MiniMax-M3", + base_url="https://api.minimax.io/anthropic", + defaults={"minimax": {"default_headers": headers}}, + ) + setup(agent, provider="minimax", api_key="sk-test") + assert mock_factory.call_args.kwargs["default_headers"] == headers + + +def test_mimo_chat_route_does_not_forward_unsupported_constructor_kwargs(tmp_path): + with patch("lingtai.services.vision.create_vision_service") as mock_factory: + mock_factory.return_value = MagicMock(spec=VisionService) + agent = make_provider_agent( + tmp_path, + provider="mimo", + model="mimo-v2.5", + base_url="https://mimo.example/v1", + defaults={"mimo": {"default_headers": {"X": "1"}, "wire_api": "chat_completions"}}, + ) + setup(agent, provider="mimo", api_key="sk-test") + assert mock_factory.call_args.args == ("mimo",) + assert mock_factory.call_args.kwargs == { + "api_key": "sk-test", + "model": "mimo-v2.5", + "base_url": "https://mimo.example/v1", + } + + +def test_mimo_responses_wire_is_manual_only(tmp_path): + agent = make_provider_agent( + tmp_path, + provider="mimo", + model="mimo-v2.5", + base_url="https://mimo.example/v1", + defaults={"mimo": {"wire_api": "responses"}}, + ) + mgr = setup(agent, provider="mimo", api_key="sk-test") + assert mgr._vision_service is None + assert mgr.handle({})["status"] == "error" + + def test_vision_setup_resolves_api_key_env(tmp_path, monkeypatch): """setup() should resolve api_key_env before constructing provider services.""" monkeypatch.setenv("VISION_TEST_API_KEY", "sk-from-env") @@ -125,76 +365,481 @@ def test_vision_setup_resolves_api_key_env(tmp_path, monkeypatch): mock_svc = MagicMock(spec=VisionService) mock_factory.return_value = mock_svc - agent = make_mock_agent(tmp_path) + agent = make_provider_agent( + tmp_path, + provider="zhipu", + model="GLM-5.2", + base_url="https://open.bigmodel.cn/api/coding/paas/v4", + ) mgr = setup(agent, provider="zhipu", api_key_env="VISION_TEST_API_KEY") mock_factory.assert_called_once() - assert mock_factory.call_args.args == ("zhipu",) + assert mock_factory.call_args.args == ("openai",) assert mock_factory.call_args.kwargs["api_key"] == "sk-from-env" + assert mock_factory.call_args.kwargs["model"] == "GLM-5.2" assert isinstance(mgr, VisionManager) -def test_vision_setup_with_codex_provider_without_api_key(tmp_path): - """Codex vision uses ChatGPT OAuth, so setup should not require api_key.""" +def test_codex_vision_without_explicit_current_oauth_identity_is_manual_only(tmp_path): + """Codex must not silently open the legacy default OAuth account.""" with patch("lingtai.services.vision.create_vision_service") as mock_factory: - mock_svc = MagicMock(spec=VisionService) - mock_factory.return_value = mock_svc + agent = make_provider_agent( + tmp_path, + provider="codex", + model="gpt-5.6-sol", + base_url=None, + ) + mgr = setup(agent, provider="codex") + mock_factory.assert_not_called() + assert mgr._vision_service is None + assert "no explicit current OAuth identity" in mgr._manual_reason + + +@pytest.mark.parametrize("provider", ["codex", "codex-pool", "codex_pool"]) +def test_codex_family_vision_aliases_use_codex_service(tmp_path, provider): + """All current Codex-family aliases construct the native Codex service path.""" + selection = None if provider == "codex" else { + "auth_path": "/tmp/codex-pool.json", + "selection": {"source_index": 0}, + } + defaults = ( + {"codex": {"codex_auth_path": "/tmp/codex-direct.json"}} + if provider == "codex" + else None + ) + with patch("lingtai.services.vision.create_vision_service") as mock_factory, patch( + "lingtai.auth.codex_pool.select_codex_pool_auth", return_value=selection + ) as mock_select: + mock_factory.return_value = MagicMock(spec=VisionService) + agent = make_provider_agent( + tmp_path, + provider=provider, + model="gpt-5.6-sol", + base_url=None, + defaults=defaults, + ) + setup(agent, provider=provider) + assert mock_factory.call_args.args == ("codex",) + assert mock_factory.call_args.kwargs["api_key"] is None + assert mock_factory.call_args.kwargs["model"] == "gpt-5.6-sol" + if provider == "codex": + mock_select.assert_not_called() + assert mock_factory.call_args.kwargs["token_path"] == "/tmp/codex-direct.json" + else: + mock_select.assert_called_once_with({}, model="gpt-5.6-sol") + assert mock_factory.call_args.kwargs["token_path"] == "/tmp/codex-pool.json" + + +def test_codex_vision_inherits_active_model_and_endpoint(tmp_path): + with patch("lingtai.services.vision.create_vision_service") as mock_factory: + mock_factory.return_value = MagicMock(spec=VisionService) agent = make_mock_agent(tmp_path) + agent.service.provider = "codex" + agent.service._model = "gpt-5.6-sol" + agent.service._base_url = "https://codex.example/backend-api/codex" + agent.service._provider_defaults = { + "codex": {"codex_auth_path": "/tmp/codex-current.json"} + } + setup(agent, provider="codex") + kwargs = mock_factory.call_args.kwargs + assert kwargs["model"] == "gpt-5.6-sol" + assert kwargs["base_url"] == "https://codex.example/backend-api/codex" + assert kwargs["token_path"] == "/tmp/codex-current.json" + + +def test_codex_vision_does_not_inherit_non_codex_model(tmp_path): + with patch("lingtai.services.vision.create_vision_service") as mock_factory: + agent = make_mock_agent(tmp_path) + agent.service.provider = "gemini" + agent.service._model = "gemini-2.5-pro" + agent.service._base_url = "https://generativelanguage.example" mgr = setup(agent, provider="codex") + mock_factory.assert_not_called() + assert mgr._vision_service is None + assert "no resolved current model" in mgr._manual_reason - mock_factory.assert_called_once_with("codex", api_key=None) - assert isinstance(mgr, VisionManager) +def test_direct_codex_vision_uses_configured_auth_path(tmp_path): + with patch("lingtai.services.vision.create_vision_service") as mock_factory: + mock_factory.return_value = MagicMock(spec=VisionService) + agent = make_mock_agent(tmp_path) + agent.service.provider = "codex" + agent.service._model = "gpt-5.6-sol" + agent.service._base_url = None + agent.service._provider_defaults = {"codex": {"codex_auth_path": "/tmp/codex-a.json"}} + setup(agent, provider="codex") + assert mock_factory.call_args.kwargs["model"] == "gpt-5.6-sol" + assert mock_factory.call_args.kwargs["token_path"] == "/tmp/codex-a.json" + + +def test_codex_pool_vision_selects_exact_model_and_passes_result(tmp_path): + selected = {"auth_path": "/tmp/codex-b.json", "selection": {"source_index": 1}} + with patch("lingtai.services.vision.create_vision_service") as mock_factory, patch( + "lingtai.auth.codex_pool.select_codex_pool_auth", return_value=selected + ) as mock_select: + mock_factory.return_value = MagicMock(spec=VisionService) + agent = make_mock_agent(tmp_path) + agent.service.provider = "codex-pool" + agent.service._model = "gpt-5.6-terra" + agent.service._base_url = "https://codex-pool.example/backend-api/codex" + agent.service._provider_defaults = {"codex-pool": {"codex_auth_pool_path": "pool.json"}} + setup(agent, provider="codex-pool") + mock_select.assert_called_once_with( + {"codex_auth_pool_path": "pool.json"}, model="gpt-5.6-terra" + ) + assert mock_factory.call_args.kwargs["model"] == "gpt-5.6-terra" + assert mock_factory.call_args.kwargs["base_url"] == "https://codex-pool.example/backend-api/codex" + assert mock_factory.call_args.kwargs["token_path"] == "/tmp/codex-b.json" + + +@pytest.mark.parametrize( + ("provider", "model", "base_url", "expects_base_url"), + [ + ("openai", "gpt-4.1", "https://openai.example/v1", True), + ("anthropic", "claude-sonnet-4-20250514", "https://anthropic.example", True), + ("gemini", "gemini-3-flash-preview", "https://gemini.example", False), + ], +) +def test_direct_native_vision_inherits_same_provider_model_and_endpoint( + tmp_path, provider, model, base_url, expects_base_url +): + """Direct-native vision keeps the active provider identity when providers match.""" + with patch("lingtai.services.vision.create_vision_service") as mock_factory: + mock_factory.return_value = MagicMock(spec=VisionService) + agent = make_provider_agent( + tmp_path, + provider=provider, + model=model, + base_url=base_url, + ) + setup(agent, provider=provider, api_key="sk-test") + + mock_factory.assert_called_once() + assert mock_factory.call_args.args == (provider,) + kwargs = mock_factory.call_args.kwargs + assert kwargs["api_key"] == "sk-test" + assert kwargs["model"] == model + if expects_base_url: + assert kwargs["base_url"] == base_url + else: + assert "base_url" not in kwargs -def test_vision_setup_unsupported_provider_skips(tmp_path): - """Unsupported providers gracefully skip and log capability_skipped. - The mock agent's `service._provider_defaults` is a MagicMock (not a dict), - so the OpenAI-compat fallback does not engage; the graceful skip path - runs instead. - """ +def test_direct_native_vision_honors_explicit_model_and_endpoint_over_active_provider(tmp_path): + """Capability kwargs remain authoritative for direct-native services.""" + with patch("lingtai.services.vision.create_vision_service") as mock_factory: + mock_factory.return_value = MagicMock(spec=VisionService) + agent = make_provider_agent( + tmp_path, + provider="openai", + model="gpt-4.1", + base_url="https://active-openai.example/v1", + ) + setup( + agent, + provider="openai", + api_key="sk-test", + model="gpt-4o", + base_url="https://vision-openai.example/v1", + ) + + kwargs = mock_factory.call_args.kwargs + assert kwargs["model"] == "gpt-4o" + assert kwargs["base_url"] == "https://vision-openai.example/v1" + + +def test_direct_native_vision_does_not_inherit_from_mismatched_provider(tmp_path): + """An explicit OpenAI route must not inherit or default Anthropic identity.""" + with patch("lingtai.services.vision.create_vision_service") as mock_factory: + agent = make_provider_agent( + tmp_path, + provider="anthropic", + model="claude-opus-4.1", + base_url="https://anthropic.example", + ) + mgr = setup(agent, provider="openai", api_key="sk-test") + + mock_factory.assert_not_called() + assert mgr._vision_service is None + assert "no resolved current model" in mgr._manual_reason + + +def test_direct_vision_inherits_same_current_credential(tmp_path): + """The active provider's own credential is part of its current identity.""" + with patch("lingtai.services.vision.create_vision_service") as mock_factory: + mock_factory.return_value = MagicMock(spec=VisionService) + agent = make_provider_agent( + tmp_path, + provider="openai", + model="gpt-5.6-sol", + base_url="https://openai.example/v1", + ) + agent.service.api_key = "sk-current" + setup(agent, provider="openai") + + assert mock_factory.call_args.kwargs["api_key"] == "sk-current" + assert mock_factory.call_args.kwargs["model"] == "gpt-5.6-sol" + + +def test_direct_vision_does_not_reuse_unrelated_current_credential(tmp_path): + """An explicit model/endpoint cannot borrow another provider's credential.""" + with patch("lingtai.services.vision.create_vision_service") as mock_factory: + agent = make_provider_agent( + tmp_path, + provider="anthropic", + model="claude-opus-4.1", + base_url="https://anthropic.example", + ) + agent.service.api_key = "sk-anthropic-current" + mgr = setup( + agent, + provider="openai", + model="gpt-5.6-sol", + base_url="https://openai.example/v1", + ) + + mock_factory.assert_not_called() + assert mgr._vision_service is None + assert "no resolved current credential" in mgr._manual_reason + + +def test_mimo_vision_preserves_current_model_and_endpoint(tmp_path): + """MiMo uses the active current identity on its supported route.""" + with patch("lingtai.services.vision.create_vision_service") as mock_factory: + mock_factory.return_value = MagicMock(spec=VisionService) + agent = make_provider_agent( + tmp_path, + provider="mimo", + model="mimo-v2.5-pro", + base_url="https://mimo-proxy.example/v1", + ) + setup(agent, provider="mimo", api_key="sk-test") + + kwargs = mock_factory.call_args.kwargs + assert kwargs["api_key"] == "sk-test" + assert kwargs["model"] == "mimo-v2.5-pro" + assert kwargs["base_url"] == "https://mimo-proxy.example/v1" + + +def test_mimo_vision_honors_explicit_model_and_endpoint(tmp_path): + with patch("lingtai.services.vision.create_vision_service") as mock_factory: + mock_factory.return_value = MagicMock(spec=VisionService) + agent = make_provider_agent( + tmp_path, + provider="mimo", + model="mimo-v2.5-pro", + base_url="https://active-mimo.example/v1", + ) + setup( + agent, + provider="mimo", + api_key="sk-test", + model="mimo-v2-omni", + base_url="https://vision-mimo.example/v1", + ) + + kwargs = mock_factory.call_args.kwargs + assert kwargs["model"] == "mimo-v2-omni" + assert kwargs["base_url"] == "https://vision-mimo.example/v1" + + +def test_minimax_vision_uses_current_anthropic_route(tmp_path): + with patch("lingtai.services.vision.create_vision_service") as mock_factory: + mock_factory.return_value = MagicMock(spec=VisionService) + agent = make_provider_agent( + tmp_path, + provider="minimax", + model="MiniMax-M3", + base_url="https://api.minimax.io/anthropic", + ) + setup(agent, provider="minimax", api_key="sk-test") + + mock_factory.assert_called_once_with("anthropic", api_key="sk-test", model="MiniMax-M3", base_url="https://api.minimax.io/anthropic") + + +def test_zhipu_vision_uses_current_openai_compatible_route(tmp_path): + with patch("lingtai.services.vision.create_vision_service") as mock_factory: + mock_factory.return_value = MagicMock(spec=VisionService) + agent = make_provider_agent( + tmp_path, + provider="zhipu", + model="GLM-5.2", + base_url="https://open.bigmodel.cn/api/coding/paas/v4", + ) + setup(agent, provider="zhipu", api_key="sk-test") + + mock_factory.assert_called_once_with( + "openai", + api_key="sk-test", + model="GLM-5.2", + base_url="https://open.bigmodel.cn/api/coding/paas/v4", + wire_api="chat_completions", + ) + + +def test_glm_vision_alias_uses_openai_compatible_route(tmp_path): + with patch("lingtai.services.vision.create_vision_service") as mock_factory: + mock_factory.return_value = MagicMock(spec=VisionService) + agent = make_provider_agent( + tmp_path, + provider="glm", + model="GLM-5.2", + base_url="https://api.z.ai/api/coding/paas/v4", + ) + setup(agent, provider="glm", api_key="sk-test") + + mock_factory.assert_called_once_with( + "openai", + api_key="sk-test", + model="GLM-5.2", + base_url="https://api.z.ai/api/coding/paas/v4", + wire_api="chat_completions", + ) + + +def test_glm_vision_alias_inherits_current_zhipu_identity(tmp_path): + """The documented GLM/Zhipu spelling pair shares one current route.""" + with patch("lingtai.services.vision.create_vision_service") as mock_factory: + mock_factory.return_value = MagicMock(spec=VisionService) + agent = make_provider_agent( + tmp_path, + provider="zhipu", + model="GLM-5.2", + base_url="https://api.z.ai/api/coding/paas/v4", + ) + agent.service.api_key = "sk-current-zhipu" + setup(agent, provider="glm") + + mock_factory.assert_called_once_with( + "openai", + api_key="sk-current-zhipu", + model="GLM-5.2", + base_url="https://api.z.ai/api/coding/paas/v4", + wire_api="chat_completions", + ) + + +@pytest.mark.parametrize( + "provider", + ["openrouter", "deepseek", "kimi", "grok", "qwen", "claude-code", "claude_code", "custom"], +) +def test_registered_adapters_remain_callable_with_manual_route(tmp_path, provider): + agent = make_provider_agent( + tmp_path, + provider=provider, + model="text-only", + base_url="https://relay.example/v1", + defaults={provider: {}}, + ) + result = setup(agent, provider=provider, api_key="sk-test") + assert isinstance(result, VisionManager) + agent.add_tool.assert_called_once() + handler = agent.add_tool.call_args.kwargs["handler"] + assert handler({"action": "manual"})["status"] in {"ok", "degraded"} + + +def test_vision_setup_unsupported_provider_keeps_manual_route(tmp_path): agent = make_mock_agent(tmp_path) result = setup(agent, provider="not-real") - assert result is CAPABILITY_UNAVAILABLE - agent.add_tool.assert_not_called() - agent._log.assert_called_with( - "capability_skipped", - capability="vision", - requested_provider="not-real", - reason="no vision support for provider 'not-real' (api_compat='')", - ) + assert isinstance(result, VisionManager) + agent.add_tool.assert_called_once() -def test_vision_setup_requires_provider_or_service(tmp_path): - """setup() without provider or service raises ValueError.""" +def test_vision_setup_without_provider_keeps_manual_route(tmp_path): agent = make_mock_agent(tmp_path) - with pytest.raises(ValueError, match="vision capability requires"): - setup(agent) + mgr = setup(agent) + assert mgr.manual()["status"] in {"ok", "degraded"} -def test_create_vision_service_codex_without_api_key(monkeypatch): - """Codex factory should not require api_key and should use OAuth manager.""" - fake_openai = SimpleNamespace(OpenAI=MagicMock()) - monkeypatch.setitem(sys.modules, "openai", fake_openai) +def test_setup_failure_retains_safe_manual_reason(tmp_path): + with patch("lingtai.services.vision.create_vision_service", side_effect=RuntimeError( + "token=secret https://user:pw@example.test/v1" + )): + agent = make_provider_agent( + tmp_path, provider="openai", model="gpt-4o", base_url="https://example.test/v1" + ) + (tmp_path / "x.png").write_bytes(b"fake") + mgr = setup(agent, provider="openai", api_key="sk-test") + result = mgr.handle({"image_path": "x.png"}) + assert result["status"] == "error" + assert "RuntimeError" in result["message"] + assert "secret" not in result["message"] + assert "example.test" not in result["message"] + - with patch("lingtai.services.vision.codex.CodexTokenManager") as mock_mgr: - svc = create_vision_service("codex") +@pytest.mark.parametrize("token_path", [None, "", " "]) +def test_create_vision_service_codex_requires_explicit_token_path(token_path): + """Codex factory must reject missing identity before importing its service.""" + kwargs = {} if token_path is None else {"token_path": token_path} + with pytest.raises(ValueError, match="token_path is required"): + create_vision_service("codex", **kwargs) + + +@pytest.mark.parametrize("token_path", [None, "", " "]) +def test_codex_vision_service_rejects_missing_token_path(token_path): + """Direct Codex construction must not bypass the explicit identity guard.""" from lingtai.services.vision.codex import CodexVisionService - assert isinstance(svc, CodexVisionService) - mock_mgr.assert_called_once_with(token_path=None) + kwargs = {} if token_path is None else {"token_path": token_path} + with pytest.raises(ValueError, match="token_path is required"): + CodexVisionService(**kwargs) + +def test_invalid_codex_direct_construction_never_imports_auth_manager(): + """A fresh interpreter must reject invalid identity before importing auth code.""" + script = """ +import sys + +assert "lingtai.auth.codex" not in sys.modules +from lingtai.services.vision.codex import CodexVisionService +assert "lingtai.auth.codex" not in sys.modules + +for kwargs in ({}, {"token_path": None}, {"token_path": ""}, {"token_path": " "}): + try: + CodexVisionService(**kwargs) + except ValueError as exc: + assert "token_path is required" in str(exc) + else: + raise AssertionError(f"invalid Codex identity was accepted: {kwargs!r}") + +assert "lingtai.auth.codex" not in sys.modules +""" + env = os.environ.copy() + env["PYTHONPATH"] = os.path.join(os.path.dirname(os.path.dirname(__file__)), "src") + env["PYTHONDONTWRITEBYTECODE"] = "1" + completed = subprocess.run( + [sys.executable, "-c", script], + env=env, + text=True, + capture_output=True, + check=False, + ) + assert completed.returncode == 0, completed.stderr -def test_create_vision_service_codex_ignores_extra_preset_kwargs(monkeypatch): - """Codex vision should tolerate irrelevant preset kwargs like api_key_env.""" + +def test_mimo_chat_completions_constructor_accepts_current_route(monkeypatch): + fake_client = MagicMock() + fake_openai = SimpleNamespace(OpenAI=MagicMock(return_value=fake_client)) + monkeypatch.setitem(sys.modules, "openai", fake_openai) + svc = create_vision_service( + "mimo", api_key="sk-test", model="mimo-v2.5", base_url="https://mimo.example/v1", max_tokens=777 + ) + from lingtai.services.vision.mimo import MiMoVisionService + assert isinstance(svc, MiMoVisionService) + fake_openai.OpenAI.assert_called_once_with(api_key="sk-test", base_url="https://mimo.example/v1") + + +def test_create_vision_service_codex_uses_explicit_path_and_filters_extra_kwargs(monkeypatch): + """Codex vision keeps the explicit path while ignoring preset-only kwargs.""" fake_openai = SimpleNamespace(OpenAI=MagicMock()) monkeypatch.setitem(sys.modules, "openai", fake_openai) - with patch("lingtai.services.vision.codex.CodexTokenManager") as mock_mgr: + with patch("lingtai.auth.codex.CodexTokenManager") as mock_mgr: svc = create_vision_service( "codex", + token_path="/tmp/codex-explicit.json", api_key_env="IGNORED", provider_note="from preset", ) @@ -202,7 +847,7 @@ def test_create_vision_service_codex_ignores_extra_preset_kwargs(monkeypatch): from lingtai.services.vision.codex import CodexVisionService assert isinstance(svc, CodexVisionService) - mock_mgr.assert_called_once_with(token_path=None) + mock_mgr.assert_called_once_with(token_path="/tmp/codex-explicit.json") def test_codex_vision_service_streams_responses_api(monkeypatch, tmp_path): @@ -222,11 +867,11 @@ def test_codex_vision_service_streams_responses_api(monkeypatch, tmp_path): openai_cls = MagicMock(return_value=client) monkeypatch.setitem(sys.modules, "openai", SimpleNamespace(OpenAI=openai_cls)) - with patch("lingtai.services.vision.codex.CodexTokenManager") as mock_mgr_cls: + with patch("lingtai.auth.codex.CodexTokenManager") as mock_mgr_cls: mock_mgr_cls.return_value.get_access_token.return_value = "oauth-token" from lingtai.services.vision.codex import CodexVisionService - svc = CodexVisionService(timeout=9.5) + svc = CodexVisionService(timeout=9.5, token_path="/tmp/codex-stream.json") result = svc.analyze_image(str(img_path), prompt="What is shown?") assert result == "A chart with candles" @@ -247,6 +892,56 @@ def test_codex_vision_service_streams_responses_api(monkeypatch, tmp_path): assert content[1]["type"] == "input_image" assert content[1]["image_url"].startswith("data:image/png;base64,") + +def test_openai_responses_vision_sends_exact_request_shape(monkeypatch, tmp_path): + img_path = tmp_path / "chart.png" + img_path.write_bytes(b"fake png bytes") + responses = MagicMock() + responses.create.return_value = SimpleNamespace(output_text="answer") + client = SimpleNamespace(responses=responses) + openai_cls = MagicMock(return_value=client) + monkeypatch.setitem(sys.modules, "openai", SimpleNamespace(OpenAI=openai_cls)) + from lingtai.services.vision.openai import OpenAIVisionService + + svc = OpenAIVisionService( + api_key="sk-test", model="gpt-5.5", base_url="https://relay.example/v1", + max_tokens=321, default_headers={"X-Preset": "active"}, wire_api="responses", + ) + assert svc.analyze_image(str(img_path), prompt="Read this") == "answer" + openai_cls.assert_called_once_with( + api_key="sk-test", base_url="https://relay.example/v1", default_headers={"X-Preset": "active"} + ) + kwargs = responses.create.call_args.kwargs + assert kwargs["model"] == "gpt-5.5" + assert kwargs["max_output_tokens"] == 321 + assert set(kwargs) == {"model", "max_output_tokens", "input"} + assert kwargs["input"][0]["content"][0] == {"type": "input_text", "text": "Read this"} + assert kwargs["input"][0]["content"][1]["type"] == "input_image" + + +def test_openai_vision_rejects_unknown_wire_before_client_construction(monkeypatch): + responses = MagicMock() + chat = MagicMock() + client = SimpleNamespace( + responses=responses, + chat=SimpleNamespace(completions=chat), + ) + openai_cls = MagicMock(return_value=client) + monkeypatch.setitem(sys.modules, "openai", SimpleNamespace(OpenAI=openai_cls)) + from lingtai.services.vision.openai import OpenAIVisionService + + with pytest.raises(ValueError, match="Unsupported OpenAI vision wire"): + OpenAIVisionService( + api_key="sk-test", + model="gpt-5.5", + wire_api="unproven_wire", + ) + + openai_cls.assert_not_called() + responses.create.assert_not_called() + chat.create.assert_not_called() + + def test_create_vision_service_unknown_provider(): """create_vision_service should raise ValueError for unknown providers.""" with pytest.raises(ValueError, match="Unsupported vision provider"): @@ -269,11 +964,9 @@ def test_vision_empty_image_path(tmp_path): assert "image_path" in result["message"].lower() or "provide" in result["message"].lower() -def test_vision_setup_no_provider_raises(tmp_path): - """setup() without provider or service should raise ValueError.""" +def test_vision_setup_no_provider_is_manual_only(tmp_path): agent = make_mock_agent(tmp_path) - with pytest.raises(ValueError, match="vision capability requires"): - setup(agent) + assert isinstance(setup(agent), VisionManager) def make_custom_agent(tmp_path, *, api_compat=None, base_url=None, model=None): @@ -319,7 +1012,7 @@ def test_vision_fallback_anthropic_compat_routes_to_anthropic_service(tmp_path): """C-2: api_compat='anthropic' routes vision through AnthropicVisionService. Previously only the openai branch existed; anthropic-compat custom proxies - fell through to capability_skipped even though AnthropicVisionService exists. + retains a manual route even though AnthropicVisionService exists. """ with patch("lingtai.services.vision.anthropic.AnthropicVisionService") as mock_cls: agent = make_custom_agent( @@ -366,25 +1059,21 @@ def test_vision_fallback_honors_capability_kwargs_over_service(tmp_path): assert isinstance(mgr, VisionManager) -def test_vision_fallback_unknown_api_compat_skips_with_diagnostic(tmp_path): +def test_vision_fallback_unknown_api_compat_keeps_manual_route(tmp_path): """Fallback with an unhandled api_compat skips and names api_compat in the reason.""" agent = make_custom_agent(tmp_path, api_compat="gemini") result = setup(agent, provider="custom", api_key="sk-test") - assert result is CAPABILITY_UNAVAILABLE - agent.add_tool.assert_not_called() - log_kwargs = agent._log.call_args.kwargs - assert log_kwargs["capability"] == "vision" - assert log_kwargs["requested_provider"] == "custom" - assert "gemini" in log_kwargs["reason"] - assert "api_compat" in log_kwargs["reason"] + assert isinstance(result, VisionManager) + agent.add_tool.assert_called_once() + assert result.manual()["status"] in {"ok", "degraded"} -def test_minimax_vision_setup_filters_inherited_api_compat(tmp_path): +def test_minimax_vision_setup_uses_anthropic_route(tmp_path): """MiniMax vision should ignore LLM transport kwargs inherited from presets. Regression: presets.expand_inherit copies api_compat from the main LLM into - `vision: {provider: inherit}`. MiniMaxVisionService accepts api_host, not - api_compat, so setup must filter provider-specific kwargs before factory + `vision: {provider: inherit}`. The current MiniMax route is Anthropic- + compatible, so setup must filter provider transport metadata before factory construction. """ with patch("lingtai.services.vision.create_vision_service") as mock_factory: @@ -398,12 +1087,14 @@ def test_minimax_vision_setup_filters_inherited_api_compat(tmp_path): provider="minimax", api_key="sk-test", api_compat="anthropic", + model="MiniMax-M3", base_url="https://api.minimaxi.com/anthropic", ) mock_factory.assert_called_once_with( - "minimax", + "anthropic", api_key="sk-test", - api_host="https://api.minimaxi.com", + model="MiniMax-M3", + base_url="https://api.minimaxi.com/anthropic", ) assert isinstance(mgr, VisionManager) diff --git a/tests/test_vision_services.py b/tests/test_vision_services.py index 29f4957c0..515827e18 100644 --- a/tests/test_vision_services.py +++ b/tests/test_vision_services.py @@ -1,6 +1,7 @@ """Tests for provider-specific VisionService response handling (issue #114, Bug G).""" from __future__ import annotations +import builtins import sys from types import SimpleNamespace from unittest.mock import MagicMock @@ -137,3 +138,72 @@ def test_anthropic_vision_service_omits_base_url_when_unset(monkeypatch): AnthropicVisionService(api_key="sk-test") anthropic_cls.assert_called_once_with(api_key="sk-test") + + +@pytest.mark.parametrize( + ("provider", "provider_module"), + [ + ("openai", "lingtai.services.vision.openai"), + ("anthropic", "lingtai.services.vision.anthropic"), + ("gemini", "lingtai.services.vision.gemini"), + ("mimo", "lingtai.services.vision.mimo"), + ], +) +@pytest.mark.parametrize("api_key", [None, "", " \t"]) +def test_factory_rejects_blank_api_key_before_provider_import( + monkeypatch, provider, provider_module, api_key +): + """Factory credential admission must precede every API provider import.""" + monkeypatch.setenv("OPENAI_API_KEY", "ambient-sdk-sentinel") + real_import = builtins.__import__ + + def block_provider_import(name, *args, **kwargs): + if name == provider_module or name.startswith(f"{provider_module}."): + raise AssertionError(f"provider module imported before credential admission: {name}") + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", block_provider_import) + with pytest.raises(ValueError, match="api_key is required"): + from lingtai.services.vision import create_vision_service + + create_vision_service(provider, api_key=api_key) + + +@pytest.mark.parametrize( + ("provider", "sdk_name", "service_import", "service_name"), + [ + ("openai", "openai", "lingtai.services.vision.openai", "OpenAIVisionService"), + ("anthropic", "anthropic", "lingtai.services.vision.anthropic", "AnthropicVisionService"), + ("gemini", "google", "lingtai.services.vision.gemini", "GeminiVisionService"), + ("mimo", "openai", "lingtai.services.vision.mimo", "MiMoVisionService"), + ], +) +@pytest.mark.parametrize("api_key", [None, "", " \t"]) +def test_direct_service_rejects_blank_api_key_before_sdk_import( + monkeypatch, provider, sdk_name, service_import, service_name, api_key +): + """Direct constructors must not let their SDK adopt an ambient credential.""" + monkeypatch.setenv("OPENAI_API_KEY", "ambient-sdk-sentinel") + module = __import__(service_import, fromlist=[service_name]) + service_cls = getattr(module, service_name) + real_import = builtins.__import__ + + def block_sdk_import(name, *args, **kwargs): + if name == sdk_name or name.startswith(f"{sdk_name}."): + raise AssertionError(f"SDK imported before credential admission: {name}") + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", block_sdk_import) + with pytest.raises(ValueError, match="api_key is required"): + service_cls(api_key=api_key) + + +def test_factory_preserves_original_nonblank_key(monkeypatch): + """Valid keys are admitted without trimming or otherwise changing them.""" + openai_cls = MagicMock(return_value=SimpleNamespace()) + monkeypatch.setitem(sys.modules, "openai", SimpleNamespace(OpenAI=openai_cls)) + + from lingtai.services.vision import create_vision_service + + create_vision_service("openai", api_key=" sk-preserve ") + openai_cls.assert_called_once_with(api_key=" sk-preserve ") From c99e7fad9d651f68128407b23adf1839190fa1e5 Mon Sep 17 00:00:00 2001 From: huangzesen Date: Thu, 16 Jul 2026 02:17:51 -0700 Subject: [PATCH 2/2] docs(vision): explain gateway try-first behavior Document that OpenRouter/custom first attempt the current OpenAI-compatible route, then return a sanitized tool failure pointing to manual alternatives without implicit provider/model/MCP fallback. --- src/lingtai/tools/vision/CONTRACT.md | 8 ++++++-- src/lingtai/tools/vision/__init__.py | 14 ++++++++++++-- src/lingtai/tools/vision/manual/SKILL.md | 9 +++++++++ 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/src/lingtai/tools/vision/CONTRACT.md b/src/lingtai/tools/vision/CONTRACT.md index 3619e7a9a..72088858e 100644 --- a/src/lingtai/tools/vision/CONTRACT.md +++ b/src/lingtai/tools/vision/CONTRACT.md @@ -29,8 +29,12 @@ requires it and resolves relative paths against the agent working directory. `claude_code`. The local mlx-vlm pseudo-provider remains available only through explicit `add_capability(..., provider="local")` opt-in and is intentionally not advertised to wizards/check-caps. Claude Code is manual-only; Codex aliases use -native Codex Responses; MiniMax uses the Anthropic route; compatible aliases use -the current OpenAI/Anthropic identity. +native Codex Responses; MiniMax uses the Anthropic route. OpenRouter and custom +deliberately try the current OpenAI-compatible model/endpoint/credential without +preflighting image support; other compatible aliases use the current +OpenAI/Anthropic identity. A real request failure is returned as a sanitized +vision tool error that points to `vision(action="manual")` for explicit +alternatives, without silently switching model/provider or invoking MCP. ## Current identity and wires diff --git a/src/lingtai/tools/vision/__init__.py b/src/lingtai/tools/vision/__init__.py index 6f8f7011d..189fc1b98 100644 --- a/src/lingtai/tools/vision/__init__.py +++ b/src/lingtai/tools/vision/__init__.py @@ -70,7 +70,12 @@ def _effective_openai_wire( } def get_description(lang: str = "en") -> str: - return "Analyze an image with the active preset when directly supported. Use action='manual' for read-only guidance when unsupported or after a direct failure. No provider or MCP fallback is automatic." + return ( + "Analyze an image with the active preset. OpenRouter/custom first try the " + "current OpenAI-compatible route; a real request failure returns a " + "sanitized error and points to action='manual' for read-only alternatives. " + "No provider, model, credential, or MCP fallback is automatic." + ) def get_schema(lang: str = "en") -> dict: @@ -87,7 +92,12 @@ def get_schema(lang: str = "en") -> dict: "type": "string", "enum": ["analyze", "manual"], "default": "analyze", - "description": "manual returns bundled read-only vision guidance; analyze performs the direct request.", + "description": ( + "analyze performs the direct request; OpenRouter/custom try the " + "current OpenAI-compatible route even when downstream image " + "support is unknown. On real failure the sanitized tool result " + "points to manual, which returns bundled read-only alternatives." + ), }, }, "required": [], diff --git a/src/lingtai/tools/vision/manual/SKILL.md b/src/lingtai/tools/vision/manual/SKILL.md index 0c6095bb9..528d9d955 100644 --- a/src/lingtai/tools/vision/manual/SKILL.md +++ b/src/lingtai/tools/vision/manual/SKILL.md @@ -17,6 +17,15 @@ maintenance: | This is the provider-neutral fallback for `vision`. It contains guidance only; it does not discover, install, start, or invoke a backend. +## OpenRouter and custom try first + +For OpenRouter and custom OpenAI-compatible presets, `vision(action="analyze")` +first tries the current endpoint, model, and credential. It does not reject the +route merely because downstream image support cannot be known in advance. If +the real request fails, the sanitized vision tool result reports the failure +type and points here for explicit alternatives; it does not expose exception +contents or silently switch provider, model, credential, or MCP. + ## Direct route When `vision` reports a direct setup or request failure, inspect the identity