diff --git a/Sources/MLXInferenceCore/ModelArchitectureProbe.swift b/Sources/MLXInferenceCore/ModelArchitectureProbe.swift index 6c9d706..19b2d47 100644 --- a/Sources/MLXInferenceCore/ModelArchitectureProbe.swift +++ b/Sources/MLXInferenceCore/ModelArchitectureProbe.swift @@ -27,7 +27,16 @@ public enum ModelArchitectureProbe { "qwen3_5_moe", "idefics3", "gemma3", - "gemma4", + // "gemma4" deliberately absent: this codebase registers that exact model_type + // in both LLMModelFactory (MLXLLM/Gemma4.swift, text_config only, no vision + // field at all) and VLMModelFactory (MLXVLM/Gemma4.swift, vision_config + // required). The string alone cannot tell them apart — matching on it treated + // every real text-only Gemma4 checkpoint as a VLM and routed it to a factory + // whose config decoder requires a field the checkpoint never has, which is a + // hard failure, not a degraded one. The vision_config-presence check below + // already distinguishes the two correctly, since only the VLM struct requires + // that field — this entry was strictly redundant on the VLM side and wrong on + // the LLM side. "smolvlm", "fastvlm", "llava_qwen2", diff --git a/Sources/SwiftLM/Server.swift b/Sources/SwiftLM/Server.swift index f876a62..526caa8 100644 --- a/Sources/SwiftLM/Server.swift +++ b/Sources/SwiftLM/Server.swift @@ -624,7 +624,34 @@ struct MLXServer: AsyncParsableCommand { configuration: modelConfig, downloader: downloader ) - let isVision = self.vision + // architecture is probed above for both paths; the CLI previously discarded it + // here (`let isVision = self.vision`), so a known VLM checkpoint loaded as a + // plain LLM unless the user remembered `--vision` — confirmed by running + // LiquidAI/LFM2.5-VL-450M-MLX-4bit without the flag: "Loading LLM" and the + // subsequent image request failed. InferenceEngine.swift (SwiftBuddy's loader) + // already auto-detects unconditionally; this mirrors that for the CLI binary + // while keeping --vision/--audio as explicit overrides. + // Speculative/MTP decoding (--draft-model, --dflash, --mtp) only wires up + // BaseLanguageModel from LLMModelFactory — a VLMModelFactory container isn't + // a DualModelMTP/generateMTP participant. Auto-detection must not flip a + // speculative-decoding session into VLM mode just because the checkpoint + // carries a vision_config (e.g. mlx-community/Qwen3.5-*-4bit ships one even + // when used purely as a text draft/main pair): doing so crashed the server + // with a Trace/BPT trap on the first real generation request, caught by CI's + // speculative-decoding suite. --vision remains a valid explicit override. + let speculativeDecodingRequested = self.draftModel != nil || self.dflash || self.mtp + let autoDetectedVision = !self.audio && architecture.supportsVision + && !speculativeDecodingRequested + let isVision = self.vision || autoDetectedVision + if architecture.supportsVision, !self.vision, !self.audio, speculativeDecodingRequested { + print( + "[SwiftLM] Note: \(architecture.modelType ?? "unknown") reports vision support, but speculative/MTP decoding was requested; loading as a text-only LLM." + ) + } else if autoDetectedVision { + print( + "[SwiftLM] Auto-detected VLM config (\(architecture.modelType ?? "unknown")); enabling vision mode." + ) + } let container: ModelContainer // Handle getting the simple model ID string for the tracker diff --git a/scripts/make-test-fixtures.py b/scripts/make-test-fixtures.py index ee2e48c..2a6d8a5 100755 --- a/scripts/make-test-fixtures.py +++ b/scripts/make-test-fixtures.py @@ -242,7 +242,9 @@ def build_moe_nested(out): `Gemma4Configuration` decodes `text_config` and nothing else, so here the count exists *only* one level down. That makes two things real rather than decorative: the nested walk added in #112's review follow-up, and the rule that a count under - `vision_config` must not be mistaken for the language model's. + a sibling container must not be mistaken for the language model's. The decoy + container is `audio_config`, not `vision_config` — see the comment where it is + built for why. It also covers the fused-expert remap: real gemma4 checkpoints ship `experts.gate_up_proj` as one tensor that sanitize splits in half into @@ -288,8 +290,15 @@ def build_moe_nested(out): "architectures": ["Gemma4ForConditionalGeneration"], "vocab_size": VOCAB, "text_config": text_config, - # A decoy the language-model walk has to skip. - "vision_config": {"model_type": "gemma4_vision", "num_experts": 999}, + # A decoy the language-model walk has to skip. audio_config, not + # vision_config: ModelArchitectureProbe.inspect treats the mere presence of a + # vision_config key as proof of vision support, so using it here made this + # text-only fixture auto-detect as a VLM and route to VLMModelFactory, which + # has no vision weights to find — found when the SwiftLM CLI started acting on + # that probe's result instead of discarding it. audio_config is excluded from + # the expert-count walk by ModelProfiler.nonLanguageContainers the same way, + # without tripping that probe. + "audio_config": {"model_type": "gemma4_audio", "num_experts": 999}, } json.dump(cfg, open(os.path.join(out, "config.json"), "w"), indent=2) diff --git a/tests/SwiftBuddyTests/VLMTests.swift b/tests/SwiftBuddyTests/VLMTests.swift index f96bf4f..d65df3f 100644 --- a/tests/SwiftBuddyTests/VLMTests.swift +++ b/tests/SwiftBuddyTests/VLMTests.swift @@ -12,6 +12,25 @@ final class VLMTests: XCTestCase { XCTAssertTrue(found, "Output should indicate VLM is loading. Got: \(accumulated)") } + // Feature 2: a known VLM checkpoint loads as a VLM without --vision. + // + // Before this test could pass, the CLI computed the architecture probe and then + // discarded it (`let isVision = self.vision`) — confirmed by running this exact + // model without --vision and observing "Loading LLM" followed by a failed image + // request. InferenceEngine.swift (SwiftBuddy's own loader) already auto-detected + // unconditionally; the CLI now does the same while --vision/--audio remain + // explicit overrides. + func testVLM_AutoDetectsLFM25WithoutVisionFlag() async throws { + let accumulated = try await captureStartupOutput(arguments: [ + "--model", "LiquidAI/LFM2.5-VL-450M-MLX-4bit", + ], timeout: 20.0) + + XCTAssertTrue( + accumulated.contains("Auto-detected VLM config") + || accumulated.contains("Loading VLM"), + "Output should indicate VLM auto-detection/loading. Got: \(accumulated)" + ) + } private func captureStartupOutput( arguments: [String], diff --git a/tests/fixtures/moe-nested/config.json b/tests/fixtures/moe-nested/config.json index 8dc58d1..8872d5d 100644 --- a/tests/fixtures/moe-nested/config.json +++ b/tests/fixtures/moe-nested/config.json @@ -32,8 +32,8 @@ "top_k_experts": 2, "moe_intermediate_size": 32 }, - "vision_config": { - "model_type": "gemma4_vision", + "audio_config": { + "model_type": "gemma4_audio", "num_experts": 999 } } \ No newline at end of file diff --git a/tests/test-fixtures.sh b/tests/test-fixtures.sh index 7a26f2f..632a587 100755 --- a/tests/test-fixtures.sh +++ b/tests/test-fixtures.sh @@ -11,7 +11,7 @@ # kv-shared-absent #120: gemma-4-e4b shape, shared layers ship no k/v # kv-shared-present b674: gemma-4-e2b shape, shared layers ship k/v anyway # moe-nested #112: expert count nested under text_config only, with a -# decoy count under vision_config; plus the fused +# decoy count under audio_config; plus the fused # experts.gate_up_proj split that sanitize performs # # The output is gibberish by construction — the weights are random. A fixture passes @@ -139,7 +139,7 @@ if [ "$ready" -ne 1 ]; then elif grep -q "is not MoE" "$MOE_LOG"; then fail "nested expert count missed: $(grep 'is not MoE' "$MOE_LOG" | head -1)" else - pass "moe-nested: expert count found under text_config, vision decoy ignored" + pass "moe-nested: expert count found under text_config, decoy container ignored" fi cleanup