diff --git a/backend/miloco/src/miloco/admin/router.py b/backend/miloco/src/miloco/admin/router.py index 15bba7095..adedd1d09 100644 --- a/backend/miloco/src/miloco/admin/router.py +++ b/backend/miloco/src/miloco/admin/router.py @@ -933,6 +933,7 @@ def _full_omni_payload() -> dict: "api_key_masked": _mask_api_key(p.api_key), "has_key": bool(p.api_key), "active": p.label == active.label, + "extra_headers": dict(p.extra_headers or {}), } for p in m.omni_profiles ] @@ -946,6 +947,7 @@ def _full_omni_payload() -> dict: "api_key_masked": _mask_api_key(active.api_key), "has_key": True, "active": True, + "extra_headers": dict(active.extra_headers or {}), }, ) health = asdict(get_omni_circuit_breaker().snapshot()) @@ -957,6 +959,7 @@ def _full_omni_payload() -> dict: "api_key_masked": _mask_api_key(active.api_key), "has_key": bool(active.api_key), "health": health, + "extra_headers": dict(active.extra_headers or {}), }, "profiles": profiles, } @@ -969,6 +972,7 @@ def _profiles_as_dicts() -> list[dict]: "model": p.model, "base_url": p.base_url, "api_key": p.api_key, + "extra_headers": dict(p.extra_headers or {}), } for p in get_settings().model.omni_profiles ] @@ -981,6 +985,13 @@ class OmniConfigBody(BaseModel): api_key: str | None = None # 留空 = 沿用该档案原 key(不被打码值覆盖) original_label: str | None = None # 正在编辑的档案原名(支持改名/定位);None=新增 activate: bool = True # True=同时设为当前生效;False=只入列表(激活由 /activate 负责) + extra_headers: dict[str, str] = Field( + default_factory=dict, + description=( + "附加请求头(可选)。部分网关要求携带自定义头才走特定通道或计费口径," + "例如智谱 GLM Coding Plan 需要 X-Title 才计入 MCP 通道。" + ), + ) class OmniSelectBody(BaseModel): @@ -1033,7 +1044,13 @@ async def put_omni_config( raise HTTPException(status_code=409, detail=f"档案名「{label}」已存在") # 传 base_url 让 _key_by_label 校验"URL 未变才沿用旧 key",防跨 URL 复用凭证。 key = _key_by_label(orig or label, body.api_key, base_url=base_url) - entry = {"label": label, "base_url": base_url, "model": model, "api_key": key} + entry = { + "label": label, + "base_url": base_url, + "model": model, + "api_key": key, + "extra_headers": dict(body.extra_headers or {}), + } tgt = orig or label will_activate = body.activate or _label_is_active(tgt) if will_activate: @@ -1041,7 +1058,9 @@ async def put_omni_config( raise HTTPException( status_code=400, detail={"code": "no_key", "message": "未配置 API Key"} ) - result = await _probe.probe_omni(model, base_url, key) + result = await _probe.probe_omni( + model, base_url, key, extra_headers=dict(body.extra_headers or {}) + ) if not result.get("ok"): raise HTTPException(status_code=400, detail=result) if target: @@ -1081,7 +1100,12 @@ async def activate_omni_config( status_code=400, detail={"code": "no_key", "message": "未配置 API Key"}, ) - result = await _probe.probe_omni(p.model, p.base_url, p.api_key) + result = await _probe.probe_omni( + p.model, + p.base_url, + p.api_key, + extra_headers=dict(p.extra_headers or {}), + ) if not result.get("ok"): raise HTTPException(status_code=400, detail=result) update_shared_config( @@ -1091,6 +1115,7 @@ async def activate_omni_config( "model": p.model, "base_url": p.base_url, "api_key": p.api_key, + "extra_headers": dict(p.extra_headers or {}), } } ) @@ -1188,7 +1213,7 @@ async def test_omni_config( ): """用表单值(缺省回退当前已保存配置)探测配置可用性。 - OpenAI 兼容族(MiMo/Qwen)两阶段:先 GET /models 验鉴权/可达,再发一次 max_tokens=1 的 + OpenAI 兼容族(MiMo/Qwen/GLM)两阶段:先 GET /models 验鉴权/可达,再发一次 max_tokens=1 的 极简 chat 真正验证该模型可用;非 OpenAI 兼容族(Gemini 等原生协议)没有等价 GET /models 预检语义,直接走 adapter 化的 chat 探测。消耗极少量 token,不计入 miloco 用量统计。 返回 {ok, code, status, latency_ms, message}。""" @@ -1208,7 +1233,19 @@ async def test_omni_config( message="ok", data={"ok": False, "code": "no_key", "message": "未配置 API Key"}, ) - result = await _probe.probe_omni(model, base_url, api_key) + # 取匹配档案(或当前 active)的 extra_headers,探测链路与推理链路保持同一配置。 + label = (body.label or "").strip() + extra_headers: dict[str, str] = {} + if label: + for p in get_settings().model.omni_profiles: + if p.label == label: + extra_headers = dict(p.extra_headers or {}) + break + if not extra_headers: + extra_headers = dict(omni.extra_headers or {}) + result = await _probe.probe_omni( + model, base_url, api_key, extra_headers=extra_headers + ) # 测通 + 三元组精确匹配当前 active + 熔断非 ok → 主动清熔断,与 put/activate/retry # 恢复路径对齐。护栏:测别的档案 / 未保存的新配置时不动状态。 # OPEN_CONFIG 下 tick 不会自动探测(只探 OPEN_RECOVERABLE),不清则用户测通了红条仍不消失, @@ -1275,8 +1312,22 @@ async def list_omni_models( "message": "未配置 API Key", }, ) + # 取匹配档案(或当前 active)的 extra_headers,探测链路与推理链路保持同一配置。 + label = (body.label or "").strip() + extra_headers: dict[str, str] = {} + if label: + for p in get_settings().model.omni_profiles: + if p.label == label: + extra_headers = dict(p.extra_headers or {}) + break + if not extra_headers: + extra_headers = dict(get_settings().model.omni.extra_headers or {}) return NormalResponse( - code=0, message="ok", data=await _probe.fetch_models(base_url, api_key) + code=0, + message="ok", + data=await _probe.fetch_models( + base_url, api_key, extra_headers=extra_headers + ), ) @@ -1401,7 +1452,12 @@ async def retry_omni_probe(current_user: str = Depends(verify_token)): return NormalResponse(code=0, message="ok", data=_full_omni_payload()) try: - result = await _probe.probe_omni(omni.model, omni.base_url, omni.api_key) + result = await _probe.probe_omni( + omni.model, + omni.base_url, + omni.api_key, + extra_headers=dict(omni.extra_headers or {}), + ) except asyncio.CancelledError: # 客户端断开 HTTP(用户切页/关 tab/网络抖动)时 FastAPI 抛 CancelledError。 # 此前 retry_now() 已把 state 置 HALF_OPEN,若不复位则 before_call 永久短路、 diff --git a/backend/miloco/src/miloco/config/settings.py b/backend/miloco/src/miloco/config/settings.py index 899de8c7d..092fc88b3 100644 --- a/backend/miloco/src/miloco/config/settings.py +++ b/backend/miloco/src/miloco/config/settings.py @@ -161,6 +161,14 @@ class OmniModelSettings(BaseModel): default="", description="多模态模型 API Key;为空时视为未配置,插件与后端启动前校验", ) + extra_headers: dict[str, str] = Field( + default_factory=dict, + description=( + "附加请求头(可选)。部分网关要求携带自定义头才走特定通道或计费口径," + "例如智谱 GLM Coding Plan 需要 {\"X-Title\": \"...\"} 才计入 MCP 通道;" + "默认空,是否携带由用户自行决定。" + ), + ) class ModelSettings(BaseModel): @@ -711,6 +719,7 @@ def _propagate_model_omni_to_perception(self) -> "MilocoSettings": "model": self.model.omni.model, "base_url": self.model.omni.base_url, "api_key": self.model.omni.api_key, + "extra_headers": dict(self.model.omni.extra_headers), } if merged != existing: new_engine = {**self.perception.engine, "omni": merged} diff --git a/backend/miloco/src/miloco/perception/engine/config.py b/backend/miloco/src/miloco/perception/engine/config.py index 02ec9b60c..152dd623d 100644 --- a/backend/miloco/src/miloco/perception/engine/config.py +++ b/backend/miloco/src/miloco/perception/engine/config.py @@ -24,7 +24,7 @@ class InputConfig: # 在构造前设 config.input.video_short_edge 期望生效,那样会被静默忽略。 video_short_edge: int = 512 # media_resolution: 仅 Gemini 生效的「每帧视觉 token 预算」档位。""/"low" = 66 tok/帧 - # (默认、最省);"high" = 264 tok/帧(小目标/文字更清但 4× token)。mimo/qwen 忽略此字段。 + # (默认、最省);"high" = 264 tok/帧(小目标/文字更清但 4× token)。mimo/qwen/glm 忽略此字段。 # 实测:小目标清晰度主要由输入像素分辨率(video_short_edge)决定,本档位只控每帧 token 预算, # 故默认 low;identity 等细节敏感场景可经 CLI 切 high。运行时由 GeminiAdapter 实时读 settings。 media_resolution: str = "" @@ -396,6 +396,9 @@ class OmniConfig: top_p: float = 0.95 timeout: float = 30.0 stream: bool = False + # 附加请求头(可选)。部分网关要求携带自定义头才走特定通道/计费口径 + # (如智谱 GLM Coding Plan 的 X-Title MCP 流量标识);默认空。 + extra_headers: dict[str, str] = field(default_factory=dict) @dataclass diff --git a/backend/miloco/src/miloco/perception/engine/omni/omni.py b/backend/miloco/src/miloco/perception/engine/omni/omni.py index 331e7bbe4..4e31779cb 100644 --- a/backend/miloco/src/miloco/perception/engine/omni/omni.py +++ b/backend/miloco/src/miloco/perception/engine/omni/omni.py @@ -93,7 +93,7 @@ async def run_omni( edge_packet: IdentityPacket, context: OmniContext, config: OmniConfig ) -> OmniOutput: """Run Omni layer: build prompt → call model → parse response.""" - payload = build_prompt(edge_packet, context) + payload = build_prompt(edge_packet, context, adapter=get_adapter(config.model)) raw_response = await call_omni(payload, config) output = parse_omni_response(raw_response, _rule_name_to_id(context)) output.usage = extract_usage(raw_response) @@ -104,7 +104,7 @@ async def run_omni_batch( edge_packets: list[IdentityPacket], context: OmniContext, config: OmniConfig ) -> OmniOutput: """Run Omni layer for multiple devices in the same room.""" - payload = build_batch_prompt(edge_packets, context) + payload = build_batch_prompt(edge_packets, context, adapter=get_adapter(config.model)) raw_response = await call_omni(payload, config) output = parse_omni_response(raw_response, _rule_name_to_id(context)) output.usage = extract_usage(raw_response) @@ -325,6 +325,7 @@ async def _call_omni_messages( "Content-Type": "application/json", **adapter.auth_headers(api_key), "User-Agent": MILOCO_USER_AGENT, + **(config.extra_headers or {}), } try: await cb.before_call() @@ -476,7 +477,7 @@ async def run_omni_stream( on_early_suggestions: Callable[[list[Suggestion]], Awaitable[None]] | None = None, ) -> OmniOutput: """Run Omni layer with streaming — extracts actionable fields early via callbacks.""" - payload = build_stream_prompt(edge_packet, context) + payload = build_stream_prompt(edge_packet, context, adapter=get_adapter(config.model)) return await _stream_and_parse( payload, config, @@ -497,7 +498,7 @@ async def run_omni_batch_stream( on_early_suggestions: Callable[[list[Suggestion]], Awaitable[None]] | None = None, ) -> OmniOutput: """Run Omni layer for multiple devices with streaming — extracts actionable fields early.""" - payload = build_batch_stream_prompt(edge_packets, context) + payload = build_batch_stream_prompt(edge_packets, context, adapter=get_adapter(config.model)) return await _stream_and_parse( payload, config, diff --git a/backend/miloco/src/miloco/perception/engine/omni/omni_client.py b/backend/miloco/src/miloco/perception/engine/omni/omni_client.py index 15a1e05eb..37d0ef6ab 100644 --- a/backend/miloco/src/miloco/perception/engine/omni/omni_client.py +++ b/backend/miloco/src/miloco/perception/engine/omni/omni_client.py @@ -157,6 +157,7 @@ def resolve_live_omni_config(base: OmniConfig) -> OmniConfig: model=o.model, base_url=o.base_url, api_key=o.api_key or base.api_key, + extra_headers=dict(o.extra_headers), ) _maybe_reset_breaker_on_config_change(resolved) return resolved @@ -217,6 +218,7 @@ async def call_omni( "Content-Type": "application/json", **adapter.auth_headers(api_key), "User-Agent": MILOCO_USER_AGENT, + **(config.extra_headers or {}), } try: await cb.before_call() # 熔断 OPEN → 直接抛 CircuitOpenError @@ -362,7 +364,7 @@ def _build_messages(payload: dict, adapter: OmniProviderAdapter) -> list[dict]: if payload.get("video_base64"): content.append(adapter.build_video_block(payload["video_base64"], media_info)) - elif payload.get("audio_base64"): + elif payload.get("audio_base64") and adapter.supports_audio_input: content.append(adapter.build_audio_block(payload["audio_base64"], media_info)) # Crop images (from tracker) @@ -435,6 +437,7 @@ async def call_omni_stream( "Content-Type": "application/json", **adapter.auth_headers(api_key), "User-Agent": MILOCO_USER_AGENT, + **(config.extra_headers or {}), } url = adapter.endpoint(config.base_url, config.model, stream=True) diff --git a/backend/miloco/src/miloco/perception/engine/omni/probe.py b/backend/miloco/src/miloco/perception/engine/omni/probe.py index cccf40d2e..be0d4804f 100644 --- a/backend/miloco/src/miloco/perception/engine/omni/probe.py +++ b/backend/miloco/src/miloco/perception/engine/omni/probe.py @@ -66,7 +66,9 @@ async def probe_reachable(base_url: str) -> dict | None: return {"code": "http_error", "message": f"服务返回异常(HTTP {r.status_code})"} -async def fetch_models(base_url: str, api_key: str) -> dict[str, Any]: +async def fetch_models( + base_url: str, api_key: str, extra_headers: dict[str, str] | None = None +) -> dict[str, Any]: """拉取 provider 模型列表(GET /models)。 模型下拉在「选定 model 之前」拉取,没有 model 可路由 adapter,故按 base_url 判 provider: @@ -88,6 +90,7 @@ async def fetch_models(base_url: str, api_key: str) -> dict[str, Any]: if is_gemini else {"Authorization": f"Bearer {api_key}"} ) + headers.update(extra_headers or {}) try: async with httpx.AsyncClient(timeout=_TIMEOUT) as client: r = await client.get(f"{base}/models", headers=headers) @@ -179,7 +182,12 @@ async def _probe_stream_chat( return 500, 0, False, {} -async def probe_chat(model: str, base_url: str, api_key: str) -> dict[str, Any]: +async def probe_chat( + model: str, + base_url: str, + api_key: str, + extra_headers: dict[str, str] | None = None, +) -> dict[str, Any]: """极简 chat 探测(max_tokens=1)真校验模型是否可用。 走 provider adapter 生成 body,兼容不同 provider 的强制要求(Qwen 强制 @@ -210,6 +218,7 @@ async def probe_chat(model: str, base_url: str, api_key: str) -> dict[str, Any]: headers = { **adapter.auth_headers(api_key), "Content-Type": "application/json", + **(extra_headers or {}), } t0 = time.monotonic() try: @@ -330,7 +339,12 @@ async def probe_chat(model: str, base_url: str, api_key: str) -> dict[str, Any]: } -async def probe_omni(model: str, base_url: str, api_key: str) -> dict[str, Any]: +async def probe_omni( + model: str, + base_url: str, + api_key: str, + extra_headers: dict[str, str] | None = None, +) -> dict[str, Any]: """两阶段探测:GET /models 预检 → 极简 chat 真校验。 - GET /models 网络错 → unreachable @@ -351,11 +365,15 @@ async def probe_omni(model: str, base_url: str, api_key: str) -> dict[str, Any]: ) if not isinstance(get_adapter(model), OpenAICompatAdapter): - return await probe_chat(model, base, api_key) + return await probe_chat(model, base, api_key, extra_headers) try: async with httpx.AsyncClient(timeout=_TIMEOUT) as client: r = await client.get( - f"{base}/models", headers={"Authorization": f"Bearer {api_key}"} + f"{base}/models", + headers={ + "Authorization": f"Bearer {api_key}", + **(extra_headers or {}), + }, ) except Exception as e: # noqa: BLE001 return { @@ -377,4 +395,4 @@ async def probe_omni(model: str, base_url: str, api_key: str) -> dict[str, Any]: "status": r.status_code, "message": f"服务返回异常(HTTP {r.status_code})", } - return await probe_chat(model, base, api_key) + return await probe_chat(model, base, api_key, extra_headers) diff --git a/backend/miloco/src/miloco/perception/engine/omni/prompt_builder.py b/backend/miloco/src/miloco/perception/engine/omni/prompt_builder.py index d7b7e18b8..17131cbd9 100644 --- a/backend/miloco/src/miloco/perception/engine/omni/prompt_builder.py +++ b/backend/miloco/src/miloco/perception/engine/omni/prompt_builder.py @@ -124,43 +124,49 @@ def build_prompt( identity_packet: IdentityPacket, context: OmniContext, label_lookup: "dict[str, str] | None" = None, + adapter: "OmniProviderAdapter | None" = None, ) -> dict: """Build the prompt payload for the omni model (single device). Args: label_lookup: person_id (UUID) → 姓名/标签 反查表,渲染 "已识别人物" 段时把 UUID 替换为人名。None 时直接渲染 person_id 字段值(与旧行为兼容)。 + adapter: 感知 provider adapter;听不见音频的 provider(如 GLM)会把 + audio 路由退回 video 路由,且 prompt 不声称本轮有音频。 Returns dict with keys: system_prompt, user_content, video_base64, media_info, crops. """ - return _build_payload([identity_packet], context, stream=False, label_lookup=label_lookup) + return _build_payload([identity_packet], context, stream=False, label_lookup=label_lookup, adapter=adapter) def build_batch_prompt( identity_packets: list[IdentityPacket], context: OmniContext, label_lookup: "dict[str, str] | None" = None, + adapter: "OmniProviderAdapter | None" = None, ) -> dict: """Build the prompt payload for multi-device omni inference (same room).""" - return _build_payload(identity_packets, context, stream=False, label_lookup=label_lookup) + return _build_payload(identity_packets, context, stream=False, label_lookup=label_lookup, adapter=adapter) def build_stream_prompt( identity_packet: IdentityPacket, context: OmniContext, label_lookup: "dict[str, str] | None" = None, + adapter: "OmniProviderAdapter | None" = None, ) -> dict: """Build prompt payload for streaming omni call (single device, speeches first).""" - return _build_payload([identity_packet], context, stream=True, label_lookup=label_lookup) + return _build_payload([identity_packet], context, stream=True, label_lookup=label_lookup, adapter=adapter) def build_batch_stream_prompt( identity_packets: list[IdentityPacket], context: OmniContext, label_lookup: "dict[str, str] | None" = None, + adapter: "OmniProviderAdapter | None" = None, ) -> dict: """Build prompt payload for streaming omni call (multi-device, speeches first).""" - return _build_payload(identity_packets, context, stream=True, label_lookup=label_lookup) + return _build_payload(identity_packets, context, stream=True, label_lookup=label_lookup, adapter=adapter) def build_query_prompt( @@ -249,7 +255,9 @@ def build_fused_payload( # audio route:无视觉信息,候选作废。与 video 同款 message 隔离(待判断规则/只读历史 # 各自独立 user 消息);本轮事实只放"当前时间 + 音频"——audio 无视频,不渲染名册/gallery/ # 待识别 track(名册的 bbox 是为"把姓名对应到视频里的人",audio 场景无意义)。 - if _resolve_route(packets) == "audio": + # provider 听不见音频(如 GLM 纯视觉模型)时退回 video 路由:帧本来就在(只是没变化), + # 至少还有画面这份真证据,且 prompt 不会声称"本轮有音频"。 + if _resolve_route(packets) == "audio" and _adapter_hears_audio(adapter): scene = SceneDescriptor(route="audio", has_identity=False, stream=False) system_prompt = build_system_prompt(scene, include_home_profile=False, camera_prompt=context.camera_prompt) ep = packets[0] @@ -341,7 +349,7 @@ def _candidate_bbox_ok( return False return True - adaptive = _maybe_encode_adaptive(packets, region_ok=_candidate_bbox_ok) + adaptive = _maybe_encode_adaptive(packets, region_ok=_candidate_bbox_ok, adapter=adapter) if adaptive is not None: video_b64, media_info = adaptive.video_b64, adaptive.media_info ref_image_jpeg = adaptive.ref_image_jpeg @@ -351,16 +359,19 @@ def bbox_remap(b: tuple[int, int, int, int]) -> tuple[int, int, int, int] | None return remap_bbox_norm_to_crop(b, _region, _frame_size) if video_b64 is None: video_b64, media_info = _encode_batch_video( - packets, short_edge=_effective_panorama_short_edge() + packets, short_edge=_effective_panorama_short_edge(), adapter=adapter ) # has_speech 只由本轮 VAD 决定:本轮真有人声(含 pending 的延续语音)→ VAD 自然过、 # 保留 speeches、模型把 拼成完整句;本轮无人声 → 剥 speeches,挂着的 # pending 半句不强行补全(否则模型会就着噪声脑补出一个完成句,正是要根除的幻觉)。 + # provider 听不见音频时 has_audio/has_speech 一并置 False:prompt 不声称本轮有音频, + # 既有机制会把 speeches / env_sounds 从 schema 和任务清单里剥掉。 + _hears = _adapter_hears_audio(adapter) scene = SceneDescriptor( route="video", has_identity=bool(candidates), stream=False, - has_audio=_batch_video_has_audio(packets), - has_speech=_batch_video_has_speech(packets), + has_audio=_batch_video_has_audio(packets, adapter) and _hears, + has_speech=_batch_video_has_speech(packets) and _hears, has_pets=_has_pets_for_scene(), identity_match_disabled=matching_moot, ) @@ -467,16 +478,22 @@ def _build_payload( stream: bool, label_lookup: "dict[str, str] | None" = None, include_home_profile: bool = True, + adapter: "OmniProviderAdapter | None" = None, ) -> dict: route = _resolve_route(packets) + # provider 听不见音频(如 GLM 纯视觉模型)时,audio 路由退回 video 路由: + # 帧本来就在(只是没变化),至少还有画面这份真证据,且 prompt 不会声称"本轮有音频"。 + if route == "audio" and not _adapter_hears_audio(adapter): + route = "video" # has_audio:video 路由下音频未过 gate 时为 False → schema 剥掉 speeches/env_sounds, # 避免模型就着画面脑补人声。audio 路由恒有音频。 # has_speech:video 路由下 VAD 判无人声时为 False → 只剥 speeches、保留 env_sounds。 - has_audio = True if route == "audio" else _batch_video_has_audio(packets) + _hears = _adapter_hears_audio(adapter) + has_audio = True if route == "audio" else (_batch_video_has_audio(packets, adapter) and _hears) # has_speech 只由本轮 VAD 决定:本轮真有人声(含 pending 的延续语音)→ VAD 自然过、 # 拼接照常;本轮无人声 → 剥 speeches,挂着的 pending 半句不强行补全(否则模型会就着 # 噪声脑补出完成句,正是要根除的幻觉)。 - has_speech = True if route == "audio" else _batch_video_has_speech(packets) + has_speech = True if route == "audio" else (_batch_video_has_speech(packets) and _hears) scene = SceneDescriptor( route=route, has_identity=False, stream=stream, has_audio=has_audio, has_speech=has_speech, @@ -500,7 +517,7 @@ def _build_payload( # (反而比不接更糟)。非生产路径不值得为它复刻 fused 的「参考图在前+说明」结构, # 恒走全景 = 字节等同本 PR 之前的行为(零回归)。 video_b64, media_info = _encode_batch_video( - packets, short_edge=_effective_panorama_short_edge() + packets, short_edge=_effective_panorama_short_edge(), adapter=adapter ) base["video_base64"] = video_b64 base["media_info"] = media_info @@ -1345,6 +1362,16 @@ def _audio_only_media_info(sample_rate: int) -> LocalMediaInfo: ) +def _adapter_hears_audio(adapter: "OmniProviderAdapter | None") -> bool: + """provider 能不能听见音频。 + + 决定的是「prompt 敢不敢声称本轮有音频」,比 supports_audio_input(只管 + input_audio 块发不发)覆盖面更广:video 路由的 mp4 音轨同样听不见。 + adapter 为 None(旧调用路径)时按支持处理,保持行为不变。 + """ + return adapter is None or adapter.supports_audio_input + + def _get_video_short_edge() -> int: try: from miloco.config import get_settings @@ -1370,14 +1397,24 @@ def _get_video_short_edge() -> int: _AUDIO_ONLY_ENABLED = True -def _packet_audio_included(ep: IdentityPacket) -> bool: +def _packet_audio_included( + ep: IdentityPacket, + adapter: "OmniProviderAdapter | None" = None, +) -> bool: """该 packet 的音频是否会被合成进 mp4:audio gate 通过即带(trigger=None 视为通过, - 兼容主动查询 / 旧路径)。speeches / env_sounds 字段的取舍与此一致——没喂音频就别问。""" + 兼容主动查询 / 旧路径)。speeches / env_sounds 字段的取舍与此一致——没喂音频就别问。 + provider 听不见音频(如 GLM 纯视觉模型)时不合成音轨:mp4 里混一段模型解码不了的 + AAC 是白花的带宽和 token 前处理开销,且与 prompt 里 has_audio=False 自相矛盾。""" + if not _adapter_hears_audio(adapter): + return False trig = ep.trigger return trig is None or trig.audio_active -def _batch_video_has_audio(packets: list[IdentityPacket]) -> bool: +def _batch_video_has_audio( + packets: list[IdentityPacket], + adapter: "OmniProviderAdapter | None" = None, +) -> bool: """video 路由最终合进 mp4 的音频是否存在。 与 ``_encode_batch_video`` 选设备口径一致(首个有 frames 的 device),据该 device 的 @@ -1386,7 +1423,7 @@ def _batch_video_has_audio(packets: list[IdentityPacket]) -> bool: """ for ep in packets: if ep.all_frames: - return _packet_audio_included(ep) + return _packet_audio_included(ep, adapter) return False @@ -1409,6 +1446,7 @@ def _batch_video_has_speech(packets: list[IdentityPacket]) -> bool: def _encode_video( identity_packet: IdentityPacket, short_edge: int = _VIDEO_SHORT_EDGE, + adapter: "OmniProviderAdapter | None" = None, ) -> tuple[str | None, LocalMediaInfo | None]: """Encode all frames + audio into mp4 video, return ``(base64, media_info)``。""" frames = identity_packet.all_frames @@ -1417,7 +1455,7 @@ def _encode_video( audio = ( identity_packet.audio_clip - if _packet_audio_included(identity_packet) + if _packet_audio_included(identity_packet, adapter) else np.empty(0, dtype=np.int16) ) return _encode_video_mp4( @@ -1665,6 +1703,7 @@ def _encode_audio_only_mp4( def _encode_batch_video( edge_packets: list[IdentityPacket], short_edge: int = _VIDEO_SHORT_EDGE, + adapter: "OmniProviderAdapter | None" = None, ) -> tuple[str | None, LocalMediaInfo | None]: """Encode video from the first device that has frames. @@ -1672,7 +1711,7 @@ def _encode_batch_video( 返回 ``(base64_str, media_info)``。 """ for ep in edge_packets: - b64, media_info = _encode_video(ep, short_edge=short_edge) + b64, media_info = _encode_video(ep, short_edge=short_edge, adapter=adapter) if b64 is not None: return b64, media_info return None, None @@ -1779,6 +1818,7 @@ def _maybe_encode_adaptive( packets: list[IdentityPacket], *, region_ok: "Callable[[tuple[int, int, int, int], tuple[int, int]], bool] | None" = None, + adapter: "OmniProviderAdapter | None" = None, ) -> "_AdaptiveResult | None": """Smart Crop 开启时算 crop 区域、编码 crop 视频 + 全景参考帧。 @@ -1891,7 +1931,7 @@ def _maybe_encode_adaptive( cse = max(1, min(cm * pano_w // cw, cm * pano_h // ch)) audio = ( ep.audio_clip - if _packet_audio_included(ep) + if _packet_audio_included(ep, adapter) else np.empty(0, dtype=np.int16) ) # fps 沿用 frame_info.fps(下采样后真实帧间隔),与全景视频一致——crop 逐帧不抽帧, diff --git a/backend/miloco/src/miloco/perception/engine/omni/provider.py b/backend/miloco/src/miloco/perception/engine/omni/provider.py index 2f9dfac91..48efcebe7 100644 --- a/backend/miloco/src/miloco/perception/engine/omni/provider.py +++ b/backend/miloco/src/miloco/perception/engine/omni/provider.py @@ -12,7 +12,7 @@ - parse_response / parse_stream_chunk —— 把 provider 响应反解析回 OpenAI 形态 ``{choices:[{message:{content}}], usage:{...}}``。 -OpenAI 兼容族(MiMo / Qwen)继承 ``OpenAICompatAdapter``,协议方法走默认实现,只覆写各自 +OpenAI 兼容族(MiMo / Qwen / GLM)继承 ``OpenAICompatAdapter``,协议方法走默认实现,只覆写各自 的 block / body 差异。Gemini 走原生 ``generateContent`` 协议(OpenAI 兼容端点不支持视频输入)。 """ @@ -43,6 +43,16 @@ class LocalMediaInfo: class OmniProviderAdapter(ABC): + """各 provider 协议适配的抽象基类(见模块 docstring 的 adapter 职责清单)。""" + + # provider 是否接受 ``input_audio`` 块。``False`` 时: + # 1. audio-only 窗口退回 video 路由——帧本来就在(只是没变化),改送画面 + # 而非降级为 text-only(见 prompt_builder._build_payload / build_fused_payload); + # 2. video 路由下 has_audio / has_speech 一并置 False,schema 剥掉 + # speeches / env_sounds,prompt 不声称本轮有音频; + # 3. messages 组装层兜底不拼 audio 块(omni_client._build_messages)。 + # MiMo / Qwen / Gemini 默认支持,GLM 关闭。 + supports_audio_input: bool = True @abstractmethod def build_video_block(self, video_base64: str, media: LocalMediaInfo) -> dict[str, Any]: @@ -161,6 +171,23 @@ def build_request_body( return body +class GlmAdapter(MiMoAdapter): + """GLM-4.6V API adapter(智谱开放平台,OpenAI 兼容协议)。 + + 鉴权与请求体与 MiMoAdapter 一致(video_url + fps + media_resolution、 + thinking:disabled 实测兼容);**不支持音频输入**——GLM-4.6V 是纯视觉语言 + 模型(输入模态为文本/图片/视频/文件),``input_audio`` 属于 GLM-4-Voice / + GLM-Realtime 线,发给 4.6V 会被 400 拒。故 ``supports_audio_input = False``, + audio-only 窗口退回 video 路由(帧本来就在,改送画面而非降级为 text-only)。 + + 若使用智谱 GLM Coding Plan 签发的 Key,需要携带 ``X-Title`` MCP 流量标识头 + (否则网关按普通资源包计费并返回 429 余额不足)——请通过配置的 + ``model.omni.extra_headers`` 自行添加,本 adapter 不做硬编码。 + """ + + supports_audio_input = False + + class QwenOmniAdapter(OpenAICompatAdapter): """Qwen3.5-Omni 系列 API adapter(qwen3.5-omni-plus / qwen3.5-omni-flash)。 @@ -461,6 +488,7 @@ def adjust_fps_for_omni(fps: int, omni_fps: int) -> int: _DEFAULT_ADAPTER = MiMoAdapter() +_GLM_ADAPTER = GlmAdapter() _QWEN_ADAPTER = QwenOmniAdapter() _GEMINI_ADAPTER = GeminiAdapter() @@ -471,6 +499,10 @@ def get_adapter(model: str) -> OmniProviderAdapter: Qwen 侧仅支持 Qwen3.5-Omni 系列(qwen3.5-omni-plus / qwen3.5-omni-flash), 旧版 qwen3-omni-flash 不支持多模态组合输入,无法满足 fused 模式需求。 + GLM 走 OpenAI 兼容协议(glm-4.6v 等),请求体与 MiMo 同构,鉴权头亦相同 + (均为标准 Bearer);唯一差异是 ``supports_audio_input = False`` + (GLM-4.6V 是纯视觉模型,不接受音频输入)。 + Gemini 走原生 generateContent 协议(OpenAI 兼容端点不支持视频输入)。 """ name = model.lower() @@ -478,4 +510,6 @@ def get_adapter(model: str) -> OmniProviderAdapter: return _QWEN_ADAPTER if "gemini" in name: return _GEMINI_ADAPTER + if "glm" in name: + return _GLM_ADAPTER return _DEFAULT_ADAPTER diff --git a/backend/miloco/src/miloco/perception/processor.py b/backend/miloco/src/miloco/perception/processor.py index 0c23e4558..04b0511e8 100644 --- a/backend/miloco/src/miloco/perception/processor.py +++ b/backend/miloco/src/miloco/perception/processor.py @@ -104,7 +104,12 @@ async def _run_omni_probe() -> None: ClassifiedError("no_key", "未配置 API Key", ErrorCategory.CONFIG), ) return - result = await _probe.probe_omni(omni.model, omni.base_url, omni.api_key) + result = await _probe.probe_omni( + omni.model, + omni.base_url, + omni.api_key, + extra_headers=dict(omni.extra_headers or {}), + ) if result.get("ok"): await cb.record_probe_result(True, None) return diff --git a/backend/miloco/tests/perception/engine/omni/test_omni_client_circuit.py b/backend/miloco/tests/perception/engine/omni/test_omni_client_circuit.py index 31d96787d..a7f33aed3 100644 --- a/backend/miloco/tests/perception/engine/omni/test_omni_client_circuit.py +++ b/backend/miloco/tests/perception/engine/omni/test_omni_client_circuit.py @@ -84,6 +84,62 @@ def _payload() -> dict: return {"system_prompt": "sys", "user_content": "u"} +def _capture_client(captured: dict): + """捕获请求头/body 的 fake AsyncClient。""" + + class _C: + def __init__(self, *a, **k): + pass + + async def __aenter__(self): + return self + + async def __aexit__(self, *a): + return False + + async def post(self, url, headers=None, json=None, **k): + captured["url"] = url + captured["headers"] = headers or {} + captured["json"] = json or {} + return _FakeResp(200, {"choices": [], "usage": {}}) + + return _C + + +async def test_call_omni_sends_extra_headers(monkeypatch): + """model.omni.extra_headers 应合并进每个请求头(GLM Coding Plan X-Title 用例)。""" + import miloco.perception.engine.omni.omni_client as oc + + captured: dict = {} + monkeypatch.setattr(oc.httpx, "AsyncClient", _capture_client(captured)) + cfg = OmniConfig( + model="glm-4.6v", + base_url="https://open.bigmodel.cn/api/paas/v4", + api_key="sk-glm", + temperature=0, + top_p=1, + max_completion_tokens=1, + timeout=1.0, + extra_headers={"X-Title": "4.5V MCP Local"}, + ) + await oc.call_omni(_payload(), cfg) + + assert captured["headers"]["Authorization"] == "Bearer sk-glm" + assert captured["headers"]["X-Title"] == "4.5V MCP Local" + + +async def test_call_omni_no_extra_headers_by_default(monkeypatch): + """未配置 extra_headers 时请求头不含额外自定义头。""" + import miloco.perception.engine.omni.omni_client as oc + + captured: dict = {} + monkeypatch.setattr(oc.httpx, "AsyncClient", _capture_client(captured)) + await oc.call_omni(_payload(), _cfg()) + + assert captured["headers"]["Authorization"] == "Bearer sk-1" + assert "X-Title" not in captured["headers"] + + # ─── call_omni × 熔断 ─────────────────────────────────────────────────────── @@ -232,6 +288,7 @@ class _Mo: model = "m1" base_url = "https://x/v1" api_key = "sk-NEW" + extra_headers: dict[str, str] = {} class _M: omni = _Mo() diff --git a/backend/miloco/tests/perception/engine/omni/test_probe.py b/backend/miloco/tests/perception/engine/omni/test_probe.py index c43215eaa..bfbbdb1e5 100644 --- a/backend/miloco/tests/perception/engine/omni/test_probe.py +++ b/backend/miloco/tests/perception/engine/omni/test_probe.py @@ -429,3 +429,71 @@ async def test_probe_chat_stream_429_preserves_retry_after(monkeypatch): assert r["code"] == "rate_limited" # 关键:Retry-After 被解析出来传给上层 _grow_backoff_locked assert r["retry_after_seconds"] == 45.0 + + +# ─── extra_headers 穿透 ───────────────────────────────────────────────────── + + +def _capture_client(captured: dict): + """捕获请求头/body 的 fake AsyncClient(GET + POST)。""" + + class _C: + def __init__(self, *a, **k): + pass + + async def __aenter__(self): + return self + + async def __aexit__(self, *a): + return False + + async def get(self, url, headers=None, **k): + captured["get_headers"] = headers or {} + captured["get_url"] = url + return _FakeResp(200, {"data": [{"id": "glm-4.6v"}]}) + + async def post(self, url, headers=None, json=None, **k): + captured["post_headers"] = headers or {} + captured["post_url"] = url + return _FakeResp(200, {"choices": [{"message": {"content": "pong"}}]}) + + return _C + + +async def test_fetch_models_passes_extra_headers(monkeypatch): + captured: dict = {} + monkeypatch.setattr(probe.httpx, "AsyncClient", _capture_client(captured)) + await probe.fetch_models( + "https://open.bigmodel.cn/api/paas/v4", + "sk-glm", + extra_headers={"X-Title": "4.5V MCP Local"}, + ) + assert captured["get_headers"]["Authorization"] == "Bearer sk-glm" + assert captured["get_headers"]["X-Title"] == "4.5V MCP Local" + + +async def test_probe_chat_passes_extra_headers(monkeypatch): + captured: dict = {} + monkeypatch.setattr(probe.httpx, "AsyncClient", _capture_client(captured)) + await probe.probe_chat( + "glm-4.6v", + "https://open.bigmodel.cn/api/paas/v4", + "sk-glm", + extra_headers={"X-Title": "4.5V MCP Local"}, + ) + assert captured["post_headers"]["X-Title"] == "4.5V MCP Local" + + +async def test_probe_omni_passes_extra_headers_both_stages(monkeypatch): + """GET /models 与 chat 回退两跳都要带 extra_headers(Coding Plan 429 回归锁)。""" + captured: dict = {} + monkeypatch.setattr(probe.httpx, "AsyncClient", _capture_client(captured)) + result = await probe.probe_omni( + "glm-4.6v", + "https://open.bigmodel.cn/api/paas/v4", + "sk-glm", + extra_headers={"X-Title": "4.5V MCP Local"}, + ) + assert result.get("ok") + assert captured["get_headers"]["X-Title"] == "4.5V MCP Local" + assert captured["post_headers"]["X-Title"] == "4.5V MCP Local" diff --git a/backend/miloco/tests/perception/engine/omni/test_prompt_builder.py b/backend/miloco/tests/perception/engine/omni/test_prompt_builder.py index 8b453c1d7..4c187329d 100644 --- a/backend/miloco/tests/perception/engine/omni/test_prompt_builder.py +++ b/backend/miloco/tests/perception/engine/omni/test_prompt_builder.py @@ -555,6 +555,21 @@ def test_audio_route_emits_input_audio_block(self): audio_block = next(b for b in user_blocks if b["type"] == "input_audio") assert audio_block["input_audio"]["data"].startswith("data:audio/m4a;base64,") + def test_audio_route_glm_drops_audio_block(self): + """GLM 不支持 input_audio:audio-only 窗口降级为 text-only,不发音频块。""" + from miloco.perception.engine.omni.omni_client import _build_messages + from miloco.perception.engine.omni.provider import GlmAdapter + + ep = _audio_only_packet() + payload = build_prompt(ep, OmniContext()) + messages = _build_messages(payload, GlmAdapter()) + user_blocks = messages[1]["content"] + types = [b["type"] for b in user_blocks] + + assert "input_audio" not in types + assert "video_url" not in types + assert any(b["type"] == "text" for b in user_blocks) + def test_video_route_emits_video_url_block(self): from miloco.perception.engine.omni.omni_client import _build_messages from miloco.perception.engine.omni.provider import MiMoAdapter @@ -1478,6 +1493,59 @@ def test_system_prompt_no_member_matching_leak_when_moot(self): assert self._MATCH_ONLY_MARKER in full +class TestGlmAudioFallback: + """GLM(纯视觉,不支持 input_audio)的音频路降级行为。 + + 锁的是第三轮 review 指出的核心行为:GLM 在 audio-only 窗口应退回 video 路由 + (送画面而非降级为 text-only),has_audio/has_speech 压 False,fused 路径同样降级。 + """ + + def test_audio_only_falls_back_to_video_for_glm(self): + from miloco.perception.engine.omni.prompt_builder import _build_payload + from miloco.perception.engine.omni.provider import GlmAdapter + + payload = _build_payload( + [_audio_only_packet()], OmniContext(), stream=False, adapter=GlmAdapter() + ) + assert "audio_base64" not in payload # 不再走音频路 + assert payload["video_base64"] # 改送视频帧 + assert "speeches" not in payload["system_prompt"] # 音频字段已剥掉 + + def test_audio_only_keeps_audio_route_for_mimo(self): + from miloco.perception.engine.omni.prompt_builder import _build_payload + from miloco.perception.engine.omni.provider import MiMoAdapter + + payload = _build_payload( + [_audio_only_packet()], OmniContext(), stream=False, adapter=MiMoAdapter() + ) + assert "audio_base64" in payload # MiMo 照常走音频路 + assert "video_base64" not in payload + + def test_fused_audio_falls_back_to_video_for_glm(self): + from miloco.perception.engine.omni.prompt_builder import build_fused_payload + from miloco.perception.engine.omni.provider import GlmAdapter + + payload = build_fused_payload( + packets=[_audio_only_packet()], context=OmniContext(), + candidates=[], gallery_snapshot={}, adapter=GlmAdapter(), + ) + user_blocks = payload["messages"][-1]["content"] + assert all(b["type"] != "input_audio" for b in user_blocks) + # fused 降级后同样送视频帧 + assert any(b["type"] == "video_url" for b in user_blocks) + + def test_video_route_glm_strips_audio_fields(self): + from miloco.perception.engine.omni.prompt_builder import build_fused_payload + from miloco.perception.engine.omni.provider import GlmAdapter + + payload = build_fused_payload( + packets=[_video_route_packet()], context=OmniContext(), + candidates=[], gallery_snapshot={}, adapter=GlmAdapter(), + ) + # video 路由 + GLM:prompt 不声称本轮有音频 + system_prompt = payload["messages"][0]["content"] + assert "speeches" not in system_prompt + def test_encode_video_pins_libx264_thread_count(monkeypatch): """omni mp4 编码器必须钉死 ENCODE_THREADS(走真实 add_stream 路径,非桩)。 diff --git a/backend/miloco/tests/perception/engine/omni/test_provider.py b/backend/miloco/tests/perception/engine/omni/test_provider.py index f526019da..c5b9f4bee 100644 --- a/backend/miloco/tests/perception/engine/omni/test_provider.py +++ b/backend/miloco/tests/perception/engine/omni/test_provider.py @@ -3,6 +3,7 @@ from miloco.perception.engine.omni import provider from miloco.perception.engine.omni.provider import ( GeminiAdapter, + GlmAdapter, LocalMediaInfo, MiMoAdapter, OpenAICompatAdapter, @@ -46,16 +47,24 @@ def test_gemini(self): def test_gemini_case_insensitive(self): assert isinstance(get_adapter("Gemini-3-Pro"), GeminiAdapter) + def test_glm(self): + assert isinstance(get_adapter("glm-4.6v"), GlmAdapter) + + def test_glm_case_insensitive(self): + assert isinstance(get_adapter("GLM-4.6V"), GlmAdapter) + def test_openai_compat_family(self): - # MiMo / Qwen 都归 OpenAI 兼容族;Gemini 不是。 + # MiMo / Qwen / GLM 都归 OpenAI 兼容族;Gemini 不是。 assert isinstance(get_adapter("xiaomi/mimo-v2.5"), OpenAICompatAdapter) assert isinstance(get_adapter("qwen3.5-omni-flash"), OpenAICompatAdapter) + assert isinstance(get_adapter("glm-4.6v"), OpenAICompatAdapter) assert not isinstance(get_adapter("gemini-3-flash-preview"), OpenAICompatAdapter) def test_singleton(self): assert get_adapter("xiaomi/mimo-v2.5") is get_adapter("xiaomi/mimo-v2.5") assert get_adapter("qwen3.5-omni-flash") is get_adapter("qwen3.5-omni-plus") assert get_adapter("gemini-3-flash") is get_adapter("gemini-3-pro") + assert get_adapter("glm-4.6v") is get_adapter("glm-4.6v-flash") class TestMiMoAdapter: @@ -92,6 +101,37 @@ def test_request_body_stream(self): assert body["thinking"] == {"type": "disabled"} +class TestGlmAdapter: + adapter = GlmAdapter() + + def test_video_block_same_as_mimo(self): + block = self.adapter.build_video_block("AAAA", _VIDEO_MEDIA) + assert block["type"] == "video_url" + assert block["fps"] == 1 + assert block["media_resolution"] == "max" + assert block["video_url"]["url"].startswith("data:video/mp4;base64,") + + def test_request_body_inherits_mimo(self): + body = self.adapter.build_request_body( + _MESSAGES, model="glm-4.6v", + max_tokens=512, temperature=0.1, top_p=0.95, stream=False, + ) + assert body["thinking"] == {"type": "disabled"} + assert body["stream"] is False + + def test_auth_headers_plain_bearer(self): + # GLM 鉴权就是标准 Bearer;X-Title 这类额外头由用户经 + # model.omni.extra_headers 配置,adapter 不硬编码。 + headers = self.adapter.auth_headers("sk-test") + assert headers["Authorization"] == "Bearer sk-test" + assert "X-Title" not in headers + + def test_supports_audio_input_false(self): + # GLM-4.6V 是纯视觉模型(文本/图片/视频/文件),input_audio 属于 + # GLM-4-Voice / GLM-Realtime 线,audio-only 窗口必须降级 text-only。 + assert self.adapter.supports_audio_input is False + + class TestQwenOmniAdapter: adapter = QwenOmniAdapter() diff --git a/backend/miloco/tests/perception/test_omni_probe_tick_drive.py b/backend/miloco/tests/perception/test_omni_probe_tick_drive.py index 924346a5b..31f8995cc 100644 --- a/backend/miloco/tests/perception/test_omni_probe_tick_drive.py +++ b/backend/miloco/tests/perception/test_omni_probe_tick_drive.py @@ -36,6 +36,7 @@ class _FakeOmni: model = "m" base_url = "https://x/v1" api_key = "sk-x" + extra_headers: dict[str, str] = {} class _FakeModel: omni = _FakeOmni() @@ -62,7 +63,7 @@ async def test_tick_drive_probe_success_recovers_to_closed(monkeypatch, _mock_om """probe 成功 → 熔断从 OPEN_RECOVERABLE 回 CLOSED,感知恢复。""" from miloco.perception import processor as _processor - async def _fake_probe(model, base_url, api_key): + async def _fake_probe(model, base_url, api_key, extra_headers=None): return {"ok": True, "code": "ok", "status": 200, "latency_ms": 10} monkeypatch.setattr( @@ -99,7 +100,7 @@ async def test_tick_drive_probe_failure_grows_backoff(monkeypatch, _mock_omni_co """probe 继续失败 → 状态回 OPEN_RECOVERABLE,backoff 涨,in-flight 清位。""" from miloco.perception import processor as _processor - async def _fake_probe(model, base_url, api_key): + async def _fake_probe(model, base_url, api_key, extra_headers=None): return {"ok": False, "code": "unreachable", "message": "still down"} monkeypatch.setattr(