feat: auto-detect VLM checkpoints on the CLI, and fix a false positive it exposed

Sources/MLXInferenceCore/ModelArchitectureProbe.swift

@@ -27,7 +27,16 @@ public enum ModelArchitectureProbe {
         "qwen3_5_moe",
         "idefics3",
         "gemma3",
-        "gemma4",
+        // "gemma4" deliberately absent: this codebase registers that exact model_type
+        // in both LLMModelFactory (MLXLLM/Gemma4.swift, text_config only, no vision
+        // field at all) and VLMModelFactory (MLXVLM/Gemma4.swift, vision_config
+        // required). The string alone cannot tell them apart — matching on it treated
+        // every real text-only Gemma4 checkpoint as a VLM and routed it to a factory
+        // whose config decoder requires a field the checkpoint never has, which is a
+        // hard failure, not a degraded one. The vision_config-presence check below
+        // already distinguishes the two correctly, since only the VLM struct requires
+        // that field — this entry was strictly redundant on the VLM side and wrong on
+        // the LLM side.
         "smolvlm",
         "fastvlm",
         "llava_qwen2",

Sources/SwiftLM/Server.swift

@@ -624,7 +624,34 @@ struct MLXServer: AsyncParsableCommand {
             configuration: modelConfig,
             downloader: downloader
         )
-        let isVision = self.vision
+        // architecture is probed above for both paths; the CLI previously discarded it
+        // here (`let isVision = self.vision`), so a known VLM checkpoint loaded as a
+        // plain LLM unless the user remembered `--vision` — confirmed by running
+        // LiquidAI/LFM2.5-VL-450M-MLX-4bit without the flag: "Loading LLM" and the
+        // subsequent image request failed. InferenceEngine.swift (SwiftBuddy's loader)
+        // already auto-detects unconditionally; this mirrors that for the CLI binary
+        // while keeping --vision/--audio as explicit overrides.
+        // Speculative/MTP decoding (--draft-model, --dflash, --mtp) only wires up
+        // BaseLanguageModel from LLMModelFactory — a VLMModelFactory container isn't
+        // a DualModelMTP/generateMTP participant. Auto-detection must not flip a
+        // speculative-decoding session into VLM mode just because the checkpoint
+        // carries a vision_config (e.g. mlx-community/Qwen3.5-*-4bit ships one even
+        // when used purely as a text draft/main pair): doing so crashed the server
+        // with a Trace/BPT trap on the first real generation request, caught by CI's
+        // speculative-decoding suite. --vision remains a valid explicit override.
+        let speculativeDecodingRequested = self.draftModel != nil || self.dflash || self.mtp
+        let autoDetectedVision = !self.audio && architecture.supportsVision
+            && !speculativeDecodingRequested
+        let isVision = self.vision || autoDetectedVision
+        if architecture.supportsVision, !self.vision, !self.audio, speculativeDecodingRequested {
+            print(
+                "[SwiftLM] Note: \(architecture.modelType ?? "unknown") reports vision support, but speculative/MTP decoding was requested; loading as a text-only LLM."
+            )
+        } else if autoDetectedVision {
+            print(
+                "[SwiftLM] Auto-detected VLM config (\(architecture.modelType ?? "unknown")); enabling vision mode."
+            )
+        }
         let container: ModelContainer
 
         // Handle getting the simple model ID string for the tracker

scripts/make-test-fixtures.py

@@ -242,7 +242,9 @@ def build_moe_nested(out):
     `Gemma4Configuration` decodes `text_config` and nothing else, so here the count
     exists *only* one level down. That makes two things real rather than decorative:
     the nested walk added in #112's review follow-up, and the rule that a count under
-    `vision_config` must not be mistaken for the language model's.
+    a sibling container must not be mistaken for the language model's. The decoy
+    container is `audio_config`, not `vision_config` — see the comment where it is
+    built for why.
 
     It also covers the fused-expert remap: real gemma4 checkpoints ship
     `experts.gate_up_proj` as one tensor that sanitize splits in half into
@@ -288,8 +290,15 @@ def build_moe_nested(out):
         "architectures": ["Gemma4ForConditionalGeneration"],
         "vocab_size": VOCAB,
         "text_config": text_config,
-        # A decoy the language-model walk has to skip.
-        "vision_config": {"model_type": "gemma4_vision", "num_experts": 999},
+        # A decoy the language-model walk has to skip. audio_config, not
+        # vision_config: ModelArchitectureProbe.inspect treats the mere presence of a
+        # vision_config key as proof of vision support, so using it here made this
+        # text-only fixture auto-detect as a VLM and route to VLMModelFactory, which
+        # has no vision weights to find — found when the SwiftLM CLI started acting on
+        # that probe's result instead of discarding it. audio_config is excluded from
+        # the expert-count walk by ModelProfiler.nonLanguageContainers the same way,
+        # without tripping that probe.
+        "audio_config": {"model_type": "gemma4_audio", "num_experts": 999},
     }
     json.dump(cfg, open(os.path.join(out, "config.json"), "w"), indent=2)
 

tests/SwiftBuddyTests/VLMTests.swift

@@ -12,6 +12,25 @@ final class VLMTests: XCTestCase {
         XCTAssertTrue(found, "Output should indicate VLM is loading. Got: \(accumulated)")
     }
 
+    // Feature 2: a known VLM checkpoint loads as a VLM without --vision.
+    //
+    // Before this test could pass, the CLI computed the architecture probe and then
+    // discarded it (`let isVision = self.vision`) — confirmed by running this exact
+    // model without --vision and observing "Loading LLM" followed by a failed image
+    // request. InferenceEngine.swift (SwiftBuddy's own loader) already auto-detected
+    // unconditionally; the CLI now does the same while --vision/--audio remain
+    // explicit overrides.
+    func testVLM_AutoDetectsLFM25WithoutVisionFlag() async throws {
+        let accumulated = try await captureStartupOutput(arguments: [
+            "--model", "LiquidAI/LFM2.5-VL-450M-MLX-4bit",
+        ], timeout: 20.0)
+
+        XCTAssertTrue(
+            accumulated.contains("Auto-detected VLM config")
+                || accumulated.contains("Loading VLM"),
+            "Output should indicate VLM auto-detection/loading. Got: \(accumulated)"
+        )
+    }
 
     private func captureStartupOutput(
         arguments: [String],

tests/fixtures/moe-nested/config.json

@@ -32,8 +32,8 @@
     "top_k_experts": 2,
     "moe_intermediate_size": 32
   },
-  "vision_config": {
-    "model_type": "gemma4_vision",
+  "audio_config": {
+    "model_type": "gemma4_audio",
     "num_experts": 999
   }
 }

tests/test-fixtures.sh

@@ -11,7 +11,7 @@
 #   kv-shared-absent    #120: gemma-4-e4b shape, shared layers ship no k/v
 #   kv-shared-present   b674:  gemma-4-e2b shape, shared layers ship k/v anyway
 #   moe-nested          #112: expert count nested under text_config only, with a
-#                       decoy count under vision_config; plus the fused
+#                       decoy count under audio_config; plus the fused
 #                       experts.gate_up_proj split that sanitize performs
 #
 # The output is gibberish by construction — the weights are random. A fixture passes
@@ -139,7 +139,7 @@ if [ "$ready" -ne 1 ]; then
 elif grep -q "is not MoE" "$MOE_LOG"; then
     fail "nested expert count missed: $(grep 'is not MoE' "$MOE_LOG" | head -1)"
 else
-    pass "moe-nested: expert count found under text_config, vision decoy ignored"
+    pass "moe-nested: expert count found under text_config, decoy container ignored"
 fi
 cleanup