Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 17 additions & 1 deletion src/lingtai/kernel/base_agent/identity.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,22 @@
import os
import platform
import sys
from urllib.parse import SplitResult, urlsplit, urlunsplit


def sanitize_endpoint(value: str) -> str:
"""Remove endpoint credentials, query parameters, and fragments for display."""
try:
parsed = urlsplit(value)
if not parsed.scheme or not parsed.netloc:
return ""
hostname = parsed.hostname or ""
host = f"[{hostname}]" if ":" in hostname and not hostname.startswith("[") else hostname
if parsed.port is not None:
host = f"{host}:{parsed.port}"
return urlunsplit(SplitResult(parsed.scheme, host, parsed.path, "", ""))
except (TypeError, ValueError):
return ""


def _set_name(agent, name: str) -> None:
Expand Down Expand Up @@ -125,7 +141,7 @@ def _safe_llm_from_service(agent) -> dict:

base_url = _effective_base_url_from_service(service)
if isinstance(base_url, str) and base_url:
llm["base_url"] = base_url
llm["base_url"] = sanitize_endpoint(base_url)

context_limit = _safe_int_attr(service, "_context_window")
if context_limit is not None:
Expand Down
69 changes: 29 additions & 40 deletions src/lingtai/services/vision/ANATOMY.md
Original file line number Diff line number Diff line change
@@ -1,71 +1,60 @@
---
related_files:
- src/lingtai/tools/vision/ANATOMY.md
- src/lingtai/services/ANATOMY.md
- src/lingtai/tools/vision/ANATOMY.md
- src/lingtai/services/vision/__init__.py
- src/lingtai/services/vision/anthropic.py
- src/lingtai/services/vision/codex.py
- src/lingtai/services/vision/gemini.py
- src/lingtai/services/vision/local.py
- src/lingtai/services/vision/mimo.py
- src/lingtai/services/vision/minimax.py
- src/lingtai/services/vision/openai.py
- src/lingtai/services/vision/zhipu.py
maintenance: |
Keep related_files as repo-relative paths to real files. Include neighboring
ANATOMY.md files so the anatomy graph stays connected rather than isolated;
anatomy links must be bidirectional. If you create a new ANATOMY.md, copy this
maintenance field. If you notice drift between this anatomy and the code,
report it. See lingtai-dev-guide for details.
anatomy links must be bidirectional. If code or citations drift, update this
map with the code change and run the architecture/document checks.
---
# src/lingtai/services/vision/

Provider-specific image understanding — standalone services that own their own API clients.

> **Maintenance:** see the `lingtai-kernel-anatomy` skill. **Coding agents** update this file in the same commit as code changes. **LingTai agents** report drift as issues.
Standalone image-understanding services. Each service owns its SDK client and
credentials; `local` is an explicit on-device pseudo-provider.

## Components

| File | LOC | Role |
|------|-----|------|
| `__init__.py` | 112 | `VisionService` ABC, `_MIME_BY_EXT` map, `_read_image()` helper, `create_vision_service()` factory |
| `anthropic.py` | 61 | `AnthropicVisionService` — base64 inline image via Anthropic Messages API |
| `codex.py` | 78 | `CodexVisionService` — ChatGPT Codex Responses API vision via OAuth token |
| `gemini.py` | 53 | `GeminiVisionService` — `genai.Client` + `types.Part.from_bytes` |
| `local.py` | 72 | `LocalVisionService` — mlx-vlm on Apple Silicon, lazy model load |
| `mimo.py` | 75 | `MiMoVisionService` — OpenAI SDK to `api.xiaomimimo.com/v1` |
| `minimax.py` | 94 | `MiniMaxVisionService` — MCP `understand_image` tool via `minimax-coding-plan-mcp` |
| `openai.py` | 56 | `OpenAIVisionService` — OpenAI chat completions with `image_url` content part |
| `zhipu.py` | 87 | `ZhipuVisionService` — MCP `analyze_image` via `@z_ai/mcp-server` Node.js subprocess |
| File | Role |
|---|---|
| `__init__.py:18-79` | `VisionService`, MIME map, image readers, and shared OpenAI-compatible message builder |
| `__init__.py:82-125` | `create_vision_service()` lazy factory for `anthropic`, `openai`, `gemini`, `mimo`, `codex`, and `local` |
| `anthropic.py:9-70` | Anthropic Messages image service; accepts active model, endpoint, headers, and token limit |
| `openai.py:7-72` | OpenAI Chat Completions or Responses image service; preserves model, endpoint, headers, wire, and output limit |
| `mimo.py:26-59` | MiMo Chat Completions service; constructor accepts only API key, model, endpoint, and token limit |
| `gemini.py:7-53` | Gemini SDK image service |
| `codex.py:11-79` | Codex Responses service using OAuth token and current model/endpoint |
| `local.py:20-72` | Local mlx-vlm pseudo-provider with lazy model loading |

## Connections

- **ABC contract** — all providers inherit `VisionService` (`__init__.py:17`); single abstract method `analyze_image(image_path, prompt) -> str`.
- **Factory** — `create_vision_service(provider, api_key=...)` at `__init__.py:63` dispatches by name with lazy imports. Supported: `anthropic`, `openai`, `gemini`, `minimax`, `zhipu`, `mimo`, `codex`, `local`.
- **MCP dependency** — `minimax.py` and `zhipu.py` import `lingtai.services.mcp.MCPClient` for subprocess-based tool calls.
- **External SDKs** — `anthropic` (Anthropic SDK), `openai` (OpenAI SDK; OpenAI/MiMo/Codex), `google.genai` (Gemini), `mlx_vlm` (local).
- **Logging** — MCP providers use `lingtai.kernel.logging.get_logger`.
- `src/lingtai/tools/vision/__init__.py` imports the factory lazily during setup.
- API services read images through `_read_image()` and encode them as required by
their wire; local passes the file path to mlx-vlm.
- OpenAI Responses uses `input_text`/`input_image` and `max_output_tokens`; Codex
retains its separate streaming Responses request shape.

## Composition

- **Standalone ownership** — each service creates and owns its own SDK client + credentials. API providers use API keys; Codex uses `CodexTokenManager`/ChatGPT OAuth; all remain independent of LLM adapters or agents.
- **Shared helper** — `_read_image(image_path) -> (bytes, mime_type)` at `__init__.py:47` used by all API-based providers.
- **MCP lazy init** — `minimax.py:34` (`_ensure_client`) and `zhipu.py:29` (`_ensure_client`) start MCP subprocesses on first call, with stale-connection recovery. Both expose `close()` for subprocess teardown.
- **Local lazy load** — `local.py:39` (`_ensure_loaded`) defers `mlx_vlm.load()` to first `analyze_image` call.
The factory dispatches only to the six services named above. MiniMax is routed
by the capability layer to Anthropic, and compatible aliases are routed to the
OpenAI or Anthropic service; no MCP vision service remains here.

## State

- **Per-call stateless** — all services read the image file fresh each call, no caching.
- **Persistent MCP clients** — `minimax._client` and `zhipu._client` hold subprocess handles across calls.
- **Local model refs** — `local._model`, `local._processor`, `local._config` persist after first load.
Services keep their client/model configuration in memory. Local keeps its lazy
model references after first load; no service writes agent working-directory
state.

## Notes

- **Image encoding** — Anthropic (`anthropic.py:34`) and OpenAI/MiMo/Codex (`openai.py:38`, `mimo.py:57`, `codex.py:45`) use base64 data URLs. Gemini (`gemini.py:37`) uses `types.Part.from_bytes`. Local passes file path directly to mlx-vlm. MCP providers send base64 in tool args (minimax) or file path (zhipu).
- **Default models** — Anthropic: `claude-sonnet-4-20250514`; Gemini: `gemini-2.5-flash`; OpenAI: `gpt-4o`; MiMo: `mimo-v2.5`; Codex: `gpt-5.5`; Local: `mlx-community/paligemma2-3b-ft-docci-448-8bit`.
- **MCP tool names** — MiniMax: `understand_image` (`minimax.py:77`); Zhipu: `analyze_image` (`zhipu.py:70`).
- **MCP launchers** — MiniMax uses `uvx minimax-coding-plan-mcp -y` (`minimax.py:64`); Zhipu uses `npx -y @z_ai/mcp-server` (`zhipu.py:58`).
- **Zhipu path vs base64** — Zhipu MCP reads the file directly by path (`zhipu.py:70`), unlike other providers that base64-encode.
- **Codex Responses API** — `codex.py` constructs `OpenAI(base_url="https://chatgpt.com/backend-api/codex")`, passes `instructions`, `stream=True`, `store=False`, `input_text` + `input_image` content blocks, and concatenates `response.output_text.delta` events. It omits `max_output_tokens` by default because the live ChatGPT Codex backend rejected that parameter (`codex.py:25-28`, `codex.py:70-71`).
- **Gemini thought filtering** — `gemini.py:51` skips `part.text` when `part.thought` is True to exclude reasoning output.
- **Git history** — 7 commits. Key: MiMo provider addition (`a728864`), zhipu path-based input fix (`2e6d53c`), region-aware ZAI/ZHIPU mode (`bed1c1e`).
The capability layer supplies the active preset model and endpoint when the
route supports them. It supplies active headers to Anthropic/OpenAI where their
constructors accept them, and never forwards headers or wire metadata to MiMo.
33 changes: 18 additions & 15 deletions src/lingtai/services/vision/__init__.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
"""VisionService — abstract image understanding backing the vision capability.

Provides standalone vision implementations for each provider that take their
own API key and handle file reading, base64 encoding, and SDK calls directly.
Provides standalone vision implementations for supported direct providers.
The local implementation is an explicit pseudo-provider and needs no API key.

Usage:
from lingtai.services.vision import VisionService, create_vision_service
Expand Down Expand Up @@ -79,11 +79,21 @@ def _image_url_messages(image_path: str, prompt: str | None = None) -> list[dict
]


def _require_api_key(api_key: object, provider: str) -> str:
"""Require a nonblank explicit API key without changing its value."""
if not isinstance(api_key, str) or not api_key.strip():
raise ValueError(
f"api_key is required for provider {provider!r}. "
f"Use provider='local' for on-device vision without an API key."
)
return api_key


def create_vision_service(provider: str, *, api_key: str | None = None, **kwargs) -> VisionService:
"""Factory — create a VisionService for the given provider.

Args:
provider: Provider name ("anthropic", "openai", "gemini", "minimax", "codex", "local").
provider: Provider name ("anthropic", "openai", "gemini", "mimo", "codex", "local").
api_key: API key for the provider (not required for "codex" or "local").
**kwargs: Additional provider-specific kwargs (e.g., model, base_url).

Expand All @@ -97,14 +107,13 @@ def create_vision_service(provider: str, *, api_key: str | None = None, **kwargs
from .local import LocalVisionService
return LocalVisionService(**kwargs)
elif provider == "codex":
token_path = kwargs.get("token_path")
if not isinstance(token_path, str) or not token_path.strip():
raise ValueError("token_path is required for Codex vision.")
from .codex import CodexVisionService
return CodexVisionService(**kwargs)

if api_key is None:
raise ValueError(
f"api_key is required for provider {provider!r}. "
f"Use provider='local' for on-device vision without an API key."
)
api_key = _require_api_key(api_key, provider)

if provider == "anthropic":
from .anthropic import AnthropicVisionService
Expand All @@ -115,17 +124,11 @@ def create_vision_service(provider: str, *, api_key: str | None = None, **kwargs
elif provider == "gemini":
from .gemini import GeminiVisionService
return GeminiVisionService(api_key=api_key, **kwargs)
elif provider == "minimax":
from .minimax import MiniMaxVisionService
return MiniMaxVisionService(api_key=api_key, **kwargs)
elif provider == "zhipu":
from .zhipu import ZhipuVisionService
return ZhipuVisionService(api_key=api_key, **kwargs)
elif provider == "mimo":
from .mimo import MiMoVisionService
return MiMoVisionService(api_key=api_key, **kwargs)
else:
raise ValueError(
f"Unsupported vision provider: {provider!r}. "
f"Supported: anthropic, openai, gemini, minimax, zhipu, mimo, codex, local."
f"Supported: anthropic, openai, gemini, mimo, codex, local."
)
6 changes: 5 additions & 1 deletion src/lingtai/services/vision/anthropic.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

import base64

from . import VisionService, _read_image
from . import VisionService, _read_image, _require_api_key


class AnthropicVisionService(VisionService):
Expand All @@ -20,14 +20,18 @@ def __init__(
model: str = "claude-sonnet-4-20250514",
base_url: str | None = None,
max_tokens: int = 1024,
default_headers: dict | None = None,
) -> None:
api_key = _require_api_key(api_key, "anthropic")
import anthropic as _anthropic

client_kwargs: dict = {"api_key": api_key}
if base_url:
# anthropic-compat local proxies (e.g. JoyCodeProxy) need an explicit
# endpoint; the SDK otherwise defaults to api.anthropic.com.
client_kwargs["base_url"] = base_url
if default_headers:
client_kwargs["default_headers"] = dict(default_headers)
self._client = _anthropic.Anthropic(**client_kwargs)
self._model = model
self._max_tokens = max_tokens
Expand Down
5 changes: 4 additions & 1 deletion src/lingtai/services/vision/codex.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
import base64
from typing import Any

from ...auth.codex import CodexTokenManager
from . import VisionService, _read_image


Expand All @@ -30,6 +29,10 @@ def __init__(
token_path: str | None = None,
**_ignored: Any,
) -> None:
if not isinstance(token_path, str) or not token_path.strip():
raise ValueError("token_path is required for Codex vision.")

from ...auth.codex import CodexTokenManager
import openai as _openai

self._openai = _openai
Expand Down
3 changes: 2 additions & 1 deletion src/lingtai/services/vision/gemini.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
"""Gemini vision service — standalone image analysis via Google's genai SDK."""
from __future__ import annotations

from . import VisionService, _read_image
from . import VisionService, _read_image, _require_api_key


class GeminiVisionService(VisionService):
Expand All @@ -17,6 +17,7 @@ def __init__(
api_key: str,
model: str = "gemini-3-flash-preview",
) -> None:
api_key = _require_api_key(api_key, "gemini")
from google import genai
from google.genai import types

Expand Down
3 changes: 2 additions & 1 deletion src/lingtai/services/vision/mimo.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
"""
from __future__ import annotations

from . import VisionService, _image_url_messages
from . import VisionService, _image_url_messages, _require_api_key


_MIMO_BASE_URL = "https://api.xiaomimimo.com/v1"
Expand All @@ -38,6 +38,7 @@ def __init__(
base_url: str | None = None,
max_tokens: int = 1024,
) -> None:
api_key = _require_api_key(api_key, "mimo")
import openai as _openai

self._client = _openai.OpenAI(
Expand Down
94 changes: 0 additions & 94 deletions src/lingtai/services/vision/minimax.py

This file was deleted.

Loading