From 5abe5767547c992f126aa15b2a8ba06039b96bfb Mon Sep 17 00:00:00 2001 From: Asher Feldman <59994+asher@users.noreply.github.com> Date: Tue, 11 Aug 2026 23:20:27 -0700 Subject: [PATCH 1/4] feat(serve): accept stream: experts on a VLM entry, placing it on the text tower, and document streamed serve as single-stream --- CHANGELOG.md | 6 +- docs/server-config.md | 13 ++-- docs/streaming.md | 14 ++++- docs/vlm.md | 11 +++- gmlx/server_bridge_vlm.py | 124 ++++++++++++++++++++++++++------------ tests/test_serving_vlm.py | 74 +++++++++++++++++++++++ 6 files changed, 191 insertions(+), 51 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c677005..429f87b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,8 +15,10 @@ adhere to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). the patch-merge projector remap onto mlx-vlm's `kimi_k25`. llama.cpp's converter writes the vision Q/K in the split 2-D RoPE layout, and the remap puts them back into MoonViT's interleaved layout. -- You can now use `--stream-experts` with `--mmproj` with the cli run/chat - commands. Server support for vision + streaming still to come. +- You can now use `--stream-experts` with `--mmproj`, on the cli run/chat + commands and on a server model entry (`stream: experts`). gmlx puts the + placement on the text tower, and the vision tower stays resident. The + server refuses `stream: cpu` on a VLM entry. ### Changed diff --git a/docs/server-config.md b/docs/server-config.md index e67888d..71ac8a3 100644 --- a/docs/server-config.md +++ b/docs/server-config.md @@ -644,10 +644,15 @@ cache stay on GPU. With the decode feeder (default, below) it matches quantized KV cache extends the advantage to long context. `stream: cpu` instead runs the whole model on the CPU device: weights stream from the page cache, so a MoE bigger than the wired-memory budget stays serveable. -Load-affecting (part of the residency identity), text-only models; rejected -on VLM and speculative/MTP entries. A `stream: cpu` entry switches the whole -process to the CPU device, so it suits a single-model server rather than -mixing with GPU-resident models. (The old key `cpu_moe: full | hybrid` is a +Load-affecting (part of the residency identity). `stream: experts` also +applies to a VLM entry. gmlx puts the placement on the text tower, and the +vision tower stays on the GPU. The server refuses `stream: cpu` on a VLM +entry, and it refuses both values on a speculative/MTP entry. A `stream: cpu` +entry switches the whole process to the CPU device, so it suits a +single-model server rather than mixing with GPU-resident models. Send only +one request at a time to a streamed entry. Concurrent requests turn off the +streaming tier's decode accelerations, which need a one-token step (see +[streaming.md](streaming.md)). (The old key `cpu_moe: full | hybrid` is a deprecated alias for `stream: cpu | experts` and warns at config load.) `moe_expert_mass: P` (a share in `(0, 1]`) installs the adaptive lossy diff --git a/docs/streaming.md b/docs/streaming.md index 1afa451..34cb98d 100644 --- a/docs/streaming.md +++ b/docs/streaming.md @@ -59,8 +59,10 @@ model's hot set. The decode feeder's exit stats (printed at `-v` on `run` and `chat`, always in server logs) show the arena hit rate a session settled at. -Streaming applies to text models - the server rejects `stream` on VLM -and speculative entries. On the CLI, MTP composes with +You can stream a text model, or the text tower of a vision model. The +server accepts `stream: experts` on a VLM entry, and puts the placement +on the text tower. The server refuses `stream: cpu` on a VLM entry. It +also refuses `stream` on a speculative entry. On the CLI, MTP composes with `--stream-experts` (not `--stream-cpu`) but defers by default: auto-MTP stays off and an explicit `--speculative` opts in. The lossy `--moe-*` levers below are hard-incompatible with MTP and force plain decoding. @@ -176,6 +178,14 @@ In server configs the placement is the per-model `stream: experts | cpu` key and the feeder opt-outs are `prefill_feeder: false` / `decode_feeder: false`. +Send only one request at a time to a server that streams a model. The +engine can batch concurrent requests, but the streaming tier makes decode +faster only when a step contains one token. The wired-memory refresh, the +lookahead prestage, and the GPU-side token path each test the step width, +and each stays off when a step contains more than one token. A second +concurrent request thus puts both requests on the slow path, and the +arena loses the hit rate that it built. This does not affect prefill. + When a larger-than-RAM model is released, its page cache is also released, via `msync(MS_INVALIDATE)` over the shards - at process exit, or at unload on a running server (`GMLX_RELEASE_PAGECACHE=0` disables). diff --git a/docs/vlm.md b/docs/vlm.md index 594e1a7..5bba391 100644 --- a/docs/vlm.md +++ b/docs/vlm.md @@ -69,9 +69,7 @@ llama.cpp's `build_rope_2d` reads. The remap puts them back into the interleaved layout of MoonViT. A mis-decoded mmproj thus gives confidently wrong image descriptions, and not a load error. GLM-5.2-V has the same vision encoder on a different text arch, and gmlx refuses it by name. K2.x is an -over-RAM MoE, so use `--stream-experts` with `--mmproj`. gmlx puts the -placement on the text tower after it loads the model, and the vision tower -stays resident. +over-RAM MoE, so it needs the streaming placement in the caveats below. Qwen2-VL / Qwen2.5-VL mmprojs (`qwen2vl_merger`) are not supported yet. The load fails up front with the family named. LLaVA's image processor isn't @@ -111,6 +109,13 @@ consumer. runs as a plain text model (the vision side is simply absent). - Adapters (`--adapter`) don't combine with `--mmproj` yet -- live GGUF LoRA is text-path-only and errors loudly. +- You can use `--stream-experts` (server key `stream: experts`) with + `--mmproj`. Use this combination for an over-RAM multimodal MoE. gmlx puts + the placement on the text tower after it loads the model, and the vision + tower stays on the GPU. You cannot use `--stream-cpu` (server key + `stream: cpu`) with `--mmproj`. That mode moves the process to the CPU + device, and it moves the vision tower with it. Send only one request at a + time to a server that streams a VLM (see [streaming.md](streaming.md)). - Speculative decoding (`--speculative`) *does* combine with `--mmproj` when a drafter is available -- a native MTP head (e.g. Qwen3.5/3.6) or `--draft-gguf`. Text-only turns speculate; media turns fall back to plain decode. With `--mmproj` but no diff --git a/gmlx/server_bridge_vlm.py b/gmlx/server_bridge_vlm.py index 2b2c27d..ff110c2 100644 --- a/gmlx/server_bridge_vlm.py +++ b/gmlx/server_bridge_vlm.py @@ -480,6 +480,61 @@ def _apply_gguf_adapter(raw_model, config, adapter_gguf: str, base_arch=base_arch) +def _install_stream_placement( + text_model, + *, + gguf_path: str, + stream, + feeder_prefill: bool | None, + feeder_decode: bool | None, + moe_experts: int | None = None, + moe_expert_mass: float | None = None, + moe_miss_shed: float | None = None, + moe_prestage: str | None = None, + moe_layer_shed: float | None = None, +) -> None: + """Put the execution placement and the lossy MoE levers on a text tower. + + ``text_model`` must be the text tower. On a VLM, use the + ``language_model``. The vision tower is small, and it stays on the GPU. + + The levers come after the placement, because each one hooks the streamed + layers that the placement makes. + """ + if stream == "cpu": + # The whole model runs on the CPU device, and this device change + # applies to the process. Use it for one over-RAM model. Do not mix + # it with GPU-resident models in one config-mode server. + from .loader import configure_stream_cpu + configure_stream_cpu( + text_model, gguf_path=gguf_path, + feeder_prefill=feeder_prefill, feeder_decode=feeder_decode) + elif stream: # "experts": routed experts stream; rest of model + KV on GPU + from .loader import install_expert_streaming + install_expert_streaming( + text_model, gguf_path=gguf_path, + feeder_prefill=feeder_prefill, feeder_decode=feeder_decode) + if moe_experts is not None: + from .loader import install_moe_experts_override + install_moe_experts_override(text_model, moe_experts) + if moe_expert_mass is not None: + from .moe_experts import install_moe_expert_mass + install_moe_expert_mass(text_model, moe_expert_mass) + if moe_miss_shed is not None: + from .moe_experts import install_moe_miss_shed + install_moe_miss_shed(text_model, moe_miss_shed) + if moe_prestage == "keepers": + if moe_miss_shed is None: + print("[stream] moe_prestage: keepers ignored: it needs " + "moe_miss_shed") + else: + from .moe_experts import install_moe_prestage_keepers + install_moe_prestage_keepers(text_model) + if moe_layer_shed is not None: + from .moe_experts import install_moe_layer_shed + install_moe_layer_shed(text_model, moe_layer_shed) + + def load_serveable_model( gguf_path: str, *, @@ -524,11 +579,13 @@ def load_serveable_model( text path (no merge, no requant). The VLM and speculative/MTP paths don't yet wire adapter apply, so an adapter on those raises rather than silently dropping it. - ``stream`` selects the text-path execution placement: ``"experts"`` streams + ``stream`` selects the execution placement: ``"experts"`` streams only the routed-expert stacks from disk while the every-token layers + KV cache stay on GPU; ``"cpu"`` runs the whole model on the CPU device (all - weights streamed through the page cache). Like the adapter, the VLM and - MTP paths raise rather than silently dropping it. + weights streamed through the page cache). ``"experts"`` also applies to a + VLM base. It goes on the text tower, and the vision tower stays on the + GPU. The MTP path, and ``"cpu"`` on a VLM, raise rather than silently + dropping it, as the adapter does. The lossy MoE levers (config ``moe_experts: K`` / ``moe_expert_mass: P`` / ``moe_miss_shed: P`` / ``moe_layer_shed: P``) install their filters/hooks @@ -538,14 +595,18 @@ def load_serveable_model( prestage through the miss-shed policy and additionally needs ``moe_miss_shed`` (announced as ignored without it). """ - def _reject_unwired(base_kind: str) -> None: + def _reject_unwired(base_kind: str, *, streamable: bool = False) -> None: # Raising beats silently dropping the option on bases that don't - # wire it yet. + # wire it yet. A streamable base accepts stream: experts, which goes + # on the text tower. It refuses stream: cpu, because that mode moves + # the process to the CPU device and moves the vision tower with it. + # A speculative base refuses both, because the engine loads the + # drafter after this function, and the drafter gets no placement. if adapter_gguf is not None: raise NotImplementedError( f"live GGUF LoRA on a {base_kind} base is not wired yet; " f"adapter={adapter_gguf!r}") - if stream: + if stream and not (streamable and stream == "experts"): raise NotImplementedError( f"stream placement on a {base_kind} base is not wired yet; " f"stream={stream!r}") @@ -563,6 +624,11 @@ def _reject_unwired(base_kind: str) -> None: ) moe_experts = moe_expert_mass = None moe_miss_shed = moe_layer_shed = moe_prestage = None + _levers = dict( + moe_experts=moe_experts, moe_expert_mass=moe_expert_mass, + moe_miss_shed=moe_miss_shed, moe_prestage=moe_prestage, + moe_layer_shed=moe_layer_shed) + if mmproj_path is not None and speculative: # VLM x MTP: text-only requests speculate; image/audio requests prefill media # into the KV and decode normally (verify is token-only over that cache). @@ -572,8 +638,14 @@ def _reject_unwired(base_kind: str) -> None: draft_gguf_path=draft_gguf_path, chat_template=chat_template) if mmproj_path is not None: - _reject_unwired("VLM") - return _load_serveable_vlm(gguf_path, mmproj_path, hf_source=hf_source) + _reject_unwired("VLM", streamable=True) + model, processor, config = _load_serveable_vlm( + gguf_path, mmproj_path, hf_source=hf_source) + _install_stream_placement( + getattr(model, "language_model", model), gguf_path=gguf_path, + stream=stream, feeder_prefill=feeder_prefill, + feeder_decode=feeder_decode, **_levers) + return model, processor, config if speculative: _reject_unwired("speculative/MTP") @@ -598,38 +670,10 @@ def _reject_unwired(base_kind: str) -> None: if adapter_gguf is not None: _apply_gguf_adapter(raw_model, config, adapter_gguf, base_gguf_path=gguf_path) - if stream == "cpu": - # Whole model on the CPU device (process-global). Intended for a single - # over-RAM positional model; mixing with GPU-resident models in one - # config-mode server is unsupported. - from .loader import configure_stream_cpu - configure_stream_cpu( - raw_model, gguf_path=gguf_path, - feeder_prefill=feeder_prefill, feeder_decode=feeder_decode) - elif stream: # "experts": routed experts stream; rest of model + KV on GPU - from .loader import install_expert_streaming - install_expert_streaming( - raw_model, gguf_path=gguf_path, - feeder_prefill=feeder_prefill, feeder_decode=feeder_decode) - if moe_experts is not None: - from .loader import install_moe_experts_override - install_moe_experts_override(raw_model, moe_experts) - if moe_expert_mass is not None: - from .moe_experts import install_moe_expert_mass - install_moe_expert_mass(raw_model, moe_expert_mass) - if moe_miss_shed is not None: - from .moe_experts import install_moe_miss_shed - install_moe_miss_shed(raw_model, moe_miss_shed) - if moe_prestage == "keepers": - if moe_miss_shed is None: - print("[stream] moe_prestage: keepers ignored: it needs " - "moe_miss_shed") - else: - from .moe_experts import install_moe_prestage_keepers - install_moe_prestage_keepers(raw_model) - if moe_layer_shed is not None: - from .moe_experts import install_moe_layer_shed - install_moe_layer_shed(raw_model, moe_layer_shed) + _install_stream_placement( + raw_model, gguf_path=gguf_path, stream=stream, + feeder_prefill=feeder_prefill, feeder_decode=feeder_decode, + **_levers) processor = _make_text_processor(tokenizer) # mlx-lm-arch caches must carry the vlm runtime's class identities or # apc/ar isinstance-gates (own classes since mlx-vlm 0.6.4) resolve diff --git a/tests/test_serving_vlm.py b/tests/test_serving_vlm.py index e3a137b..d7b34ed 100644 --- a/tests/test_serving_vlm.py +++ b/tests/test_serving_vlm.py @@ -70,6 +70,9 @@ class _FakeConfig: class _FakeVLMModel: def __init__(self): self.config = _FakeConfig() + # A real mlx-vlm Model holds its text tower here. The streaming + # placement must go on the tower, not on the wrapper. + self.language_model = object() def get_input_embeddings(self, *a, **k): # pragma: no cover - presence only raise NotImplementedError @@ -112,6 +115,77 @@ def test_vlm_branch_returns_model_processor_config(fake_load_vlm_model): "hf_source": "hf/repo", "verbose": False}] +@pytest.fixture +def spy_placement(monkeypatch): + """Record the placement/lever installers instead of running them, so the + branch is testable without a streamed model. Each entry is + ``(target, args, kwargs)`` keyed by installer name.""" + import gmlx.loader as loader + import gmlx.moe_experts as moe_experts + + seen: dict[str, tuple] = {} + + def _spy(name, module=loader): + def _fake(target, *args, **kwargs): + seen[name] = (target, args, kwargs) + monkeypatch.setattr(module, name, _fake, raising=True) + + _spy("install_expert_streaming") + _spy("configure_stream_cpu") + _spy("install_moe_experts_override") + _spy("install_moe_expert_mass", moe_experts) + _spy("install_moe_miss_shed", moe_experts) + _spy("install_moe_layer_shed", moe_experts) + return seen + + +def test_vlm_stream_experts_places_on_the_text_tower( + fake_load_vlm_model, spy_placement): + model, _proc, _config = serving.load_serveable_model( + "/m/llm.gguf", mmproj_path="/m/mmproj.gguf", stream="experts", + feeder_prefill=True, feeder_decode=False, + ) + target, _args, kwargs = spy_placement["install_expert_streaming"] + # The tower, not the VLM wrapper: the vision tower stays on the GPU. + assert target is model.language_model + assert kwargs == {"gguf_path": "/m/llm.gguf", + "feeder_prefill": True, "feeder_decode": False} + assert "configure_stream_cpu" not in spy_placement + + +def test_vlm_stream_experts_installs_the_moe_levers( + fake_load_vlm_model, spy_placement): + model, _proc, _config = serving.load_serveable_model( + "/m/llm.gguf", mmproj_path="/m/mmproj.gguf", stream="experts", + moe_experts=4, moe_expert_mass=0.8, moe_miss_shed=0.9, + moe_layer_shed=0.1, + ) + for name, value in (("install_moe_experts_override", 4), + ("install_moe_expert_mass", 0.8), + ("install_moe_miss_shed", 0.9), + ("install_moe_layer_shed", 0.1)): + target, args, _kwargs = spy_placement[name] + assert target is model.language_model + assert args == (value,) + + +def test_vlm_stream_cpu_is_refused(fake_load_vlm_model): + # stream: cpu moves the device for the process, so it moves the vision + # tower to the CPU with the text tower. + with pytest.raises(NotImplementedError, match="stream"): + serving.load_serveable_model( + "/m/llm.gguf", mmproj_path="/m/mmproj.gguf", stream="cpu") + + +def test_vlm_mtp_stream_is_refused(fake_load_vlm_model): + # The engine loads the MTP drafter after this function, and the drafter + # gets no placement. + with pytest.raises(NotImplementedError, match="VLM x MTP"): + serving.load_serveable_model( + "/m/llm.gguf", mmproj_path="/m/mmproj.gguf", stream="experts", + speculative=True) + + def test_text_branch_does_not_touch_vlm_loader(fake_load_vlm_model, monkeypatch): calls, _ = fake_load_vlm_model tokenizer = types.SimpleNamespace(eos_token_id=1) # settable; no _tokenizer From 027366933626e8de390a72985c22c98edaf747d0 Mon Sep 17 00:00:00 2001 From: Asher Feldman <59994+asher@users.noreply.github.com> Date: Wed, 12 Aug 2026 07:25:09 -0700 Subject: [PATCH 2/4] fix(vlm): evaluate the Kimi image processor's outputs on the building thread so the serve engine thread can use them --- CHANGELOG.md | 10 ++++++---- gmlx/vlm.py | 31 +++++++++++++++++++++++++++++-- tests/test_kimi_k25_vision.py | 26 ++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 429f87b..d312f7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,10 +15,12 @@ adhere to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). the patch-merge projector remap onto mlx-vlm's `kimi_k25`. llama.cpp's converter writes the vision Q/K in the split 2-D RoPE layout, and the remap puts them back into MoonViT's interleaved layout. -- You can now use `--stream-experts` with `--mmproj`, on the cli run/chat - commands and on a server model entry (`stream: experts`). gmlx puts the - placement on the text tower, and the vision tower stays resident. The - server refuses `stream: cpu` on a VLM entry. +- You can now use `--stream-experts` with `--mmproj` with the cli run/chat + commands. Server support for vision + streaming still to come. +- The server accepts `stream: experts` on a VLM entry, so you can serve an + over-RAM VLM. gmlx puts the placement on the text tower, and the vision + tower stays resident. The server refuses `stream: cpu` on a VLM entry. + Send only one request at a time to a streamed entry (see streaming.md). ### Changed diff --git a/gmlx/vlm.py b/gmlx/vlm.py index 50687b1..94c746d 100644 --- a/gmlx/vlm.py +++ b/gmlx/vlm.py @@ -1929,6 +1929,33 @@ def _synthesize_pixtral_processor(tokenizer, mm_meta: dict): return _attach_streaming_helpers(processor, tokenizer) +def _kimi_k25_image_processor(**params): + """A ``KimiK25ImageProcessor`` whose outputs are evaluated, not lazy. + + The stock processor resizes and patchifies in MLX, thus its outputs are + lazy graphs on the default stream of the thread that runs it. MLX default + streams are per-thread, and a thread cannot evaluate a lazy array from a + different thread. The server preprocesses a request on a caller thread + and evaluates on the engine thread, so the stock outputs fail there with + "There is no Stream(gpu, N) in current thread". This subclass evaluates + the outputs on the thread that builds them. The other supported families + preprocess in numpy, so they do not need this today, but mlx-vlm has many + MLX-native processors: give a family like this the same wrapper. + """ + from mlx_vlm.models.kimi_k25.processing_kimi_k25 import ( + KimiK25ImageProcessor, + ) + + class _Eager(KimiK25ImageProcessor): + def preprocess(self, images, return_tensors=None, **kwargs): + out = super().preprocess( + images, return_tensors=return_tensors, **kwargs) + mx.eval([v for v in out.values() if isinstance(v, mx.array)]) + return out + + return _Eager(**params) + + def _synthesize_kimi_k25_processor(tokenizer, mm_meta: dict): """Build the Kimi-K2.5/K2.7 processor from the GGUFs alone - no HF download. @@ -1940,7 +1967,7 @@ def _synthesize_kimi_k25_processor(tokenizer, mm_meta: dict): for it. Its value is 512, the bound of MoonViT's 2-D RoPE table. """ from mlx_vlm.models.kimi_k25.processing_kimi_k25 import ( - KimiK25ImageProcessor, KimiK25Processor, + KimiK25Processor, ) patch_size = _mm_int(mm_meta, "clip.vision.patch_size") @@ -1951,7 +1978,7 @@ def _synthesize_kimi_k25_processor(tokenizer, mm_meta: dict): in_token_limit = (int(max_pixels) // (patch_size ** 2) if max_pixels else 16384) - image_processor = KimiK25ImageProcessor( + image_processor = _kimi_k25_image_processor( patch_size=patch_size, image_mean=tuple(image_mean), image_std=tuple(image_std), diff --git a/tests/test_kimi_k25_vision.py b/tests/test_kimi_k25_vision.py index 5825871..4ed1273 100644 --- a/tests/test_kimi_k25_vision.py +++ b/tests/test_kimi_k25_vision.py @@ -217,3 +217,29 @@ def test_config_synth_without_the_placeholder_token(): cfg = _synthesize_kimi_k25_vlm_config( {"vocab_size": 10}, MM_META, {"tokenizer.ggml.tokens": ["a"]}) assert "media_placeholder_token_id" not in cfg + + +def test_image_processor_output_survives_a_thread_hop(): + """The server preprocesses a request on a caller thread and evaluates on + the engine thread. The stock KimiK25ImageProcessor returns lazy MLX + graphs, and a different thread cannot evaluate them ("There is no + Stream(gpu, N) in current thread"). The synthesized processor evaluates + its outputs on the building thread.""" + pytest.importorskip("mlx_vlm") + import threading + + from PIL import Image + + from gmlx.vlm import _kimi_k25_image_processor + + proc = _kimi_k25_image_processor( + patch_size=14, image_mean=(0.5, 0.5, 0.5), image_std=(0.5, 0.5, 0.5), + in_token_limit=1024, merge_kernel_size=[2, 2], + patch_limit_on_one_side=64) + img = Image.new("RGB", (56, 42), (200, 30, 90)) + box = {} + t = threading.Thread(target=lambda: box.update(proc.preprocess([img]))) + t.start() + t.join() + mx.eval(box["pixel_values"], box["image_grid_hws"]) + assert box["pixel_values"].shape[1:] == (3, 14, 14) From 132a8219dc3d608fbe419930a9073d7c29dff277 Mon Sep 17 00:00:00 2001 From: Asher Feldman <59994+asher@users.noreply.github.com> Date: Wed, 12 Aug 2026 18:33:38 -0700 Subject: [PATCH 3/4] test(apc): drain the async disk writer before the anchor-shadow lookup so the deep skeleton is published --- tests/test_ckpt_tier.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_ckpt_tier.py b/tests/test_ckpt_tier.py index c22a188..a0348be 100644 --- a/tests/test_ckpt_tier.py +++ b/tests/test_ckpt_tier.py @@ -1230,6 +1230,9 @@ def test_anchor_never_shadows_a_deeper_disk_skeleton(tmp_path): assert ckpt_store(man, ids[:32], shallow, extra_hash=4, kind="anchor") assert ckpt_store(man, ids[:64], deep, extra_hash=4) + # The lookup below reads the deep skeleton off disk, so the async + # writer has to have published it first. + drain_disk(disk) # Drop the deep record from memory, keeping its disk skeleton: # exactly what strip-on-extend leaves behind as a chain deepens. idx = _ckpt_records(man) From 4dd8585dc706c95ccb618e6ab6e3aeaa47b6dc72 Mon Sep 17 00:00:00 2001 From: Asher Feldman <59994+asher@users.noreply.github.com> Date: Wed, 12 Aug 2026 18:33:38 -0700 Subject: [PATCH 4/4] test: drop the dtype scoping pin and filter upstream's Pillow getdata deprecation --- tests/test_config.py | 11 ----------- tests/test_kimi_k25_vision.py | 3 +++ 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/tests/test_config.py b/tests/test_config.py index c76b3d5..16d3794 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -693,17 +693,6 @@ def test_float32_is_not_a_config_value(): build_config(doc) -def test_dtype_is_not_a_per_model_load_key(): - """The reason to leave bfloat16 is a property of the GPU, not of a model, - so `load: {dtype: ...}` is an unknown key: it warns and is dropped rather - than forking one model onto a different activation width.""" - assert "dtype" not in cfgmod.LOAD_ENV - doc = _doc() - doc["models"]["m-bare"]["overrides"] = {"load": {"dtype": "float16"}} - cfg = build_config(doc) - assert "GMLX_ACTIVATION_DTYPE" not in cfgmod.env_for(resolve_model("m-bare", cfg)) - - def test_env_for_emits_speculative_flag(): """The per-build speculative state rides the env window (the only signal that reaches the load bridge in the engine's worker thread). Always emitted - "0" diff --git a/tests/test_kimi_k25_vision.py b/tests/test_kimi_k25_vision.py index 4ed1273..89dcee1 100644 --- a/tests/test_kimi_k25_vision.py +++ b/tests/test_kimi_k25_vision.py @@ -219,6 +219,9 @@ def test_config_synth_without_the_placeholder_token(): assert "media_placeholder_token_id" not in cfg +# mlx-vlm's KimiK25ImageProcessor.to_mlx still calls Image.getdata, which +# Pillow deprecates for removal in Pillow 14. Upstream's call, not ours. +@pytest.mark.filterwarnings("ignore:Image.Image.getdata is deprecated") def test_image_processor_output_survives_a_thread_hop(): """The server preprocesses a request on a caller thread and evaluates on the engine thread. The stock KimiK25ImageProcessor returns lazy MLX