diff --git a/scripts/make-test-fixtures.py b/scripts/make-test-fixtures.py index e61ecf3..953f437 100755 --- a/scripts/make-test-fixtures.py +++ b/scripts/make-test-fixtures.py @@ -229,11 +229,89 @@ def build_gemma4_kv_shared(out, vestigial): return len(w) +def build_moe_nested(out): + """A Qwen3-style MoE: per-expert tensors, nested config, decoy vision count. + + Covers the MoE *load* path, which nothing else in CI touches — no MoE model is in + the matrix at all. Specifically: per-expert `mlp.experts.N.*` tensors being stacked + into `switch_mlp` by sanitize, and an expert count nested under `text_config` + rather than at top level. + + What it does NOT cover, despite the nesting, is the #112 detection bug itself. + That bug was `num_local_experts` being the only spelling checked, so a config + declaring `num_experts` was called dense. But `modelTypeImpliesMoE` treats any + model_type containing "moe" as MoE regardless of keys, so `qwen3_moe` is caught by + that fallback whatever the config says — verified by reverting detection to the + single-key form and watching this fixture still enable streaming. + + Reproducing #112 end to end needs a MoE architecture whose model_type does not + contain "moe" — deepseek_v3 (`n_routed_experts`) is the candidate. Its MLA + attention makes that a larger fixture; the profiler itself is covered directly by + ModelProfilerMoEDetectionTests in the meantime. + """ + H, L, HEADS, KVH, HD = 64, 2, 4, 2, 16 + INTER, MOE_INTER, EXPERTS, TOPK = 128, 32, 4, 2 + rng = np.random.default_rng(0) + + text_config = { + "model_type": "qwen3_moe", + "hidden_size": H, + "num_hidden_layers": L, + "intermediate_size": INTER, + "num_attention_heads": HEADS, + "num_key_value_heads": KVH, + "head_dim": HD, + "num_experts": EXPERTS, + "num_experts_per_tok": TOPK, + "moe_intermediate_size": MOE_INTER, + "decoder_sparse_step": 1, + "mlp_only_layers": [], + "rms_norm_eps": 1e-6, + "vocab_size": VOCAB, + "rope_theta": 10000.0, + "tie_word_embeddings": False, + "max_position_embeddings": 512, + "norm_topk_prob": True, + } + cfg = dict(text_config) + cfg["text_config"] = text_config + # Must not be mistaken for the language model's expert count. + cfg["vision_config"] = {"model_type": "qwen3_vl", "num_experts": 999} + json.dump(cfg, open(os.path.join(out, "config.json"), "w"), indent=2) + + w = { + "model.embed_tokens.weight": rand(rng, VOCAB, H), + "model.norm.weight": ones(H), + "lm_head.weight": rand(rng, VOCAB, H), + } + for i in range(L): + p_ = f"model.layers.{i}" + w[f"{p_}.self_attn.q_proj.weight"] = rand(rng, HEADS * HD, H) + w[f"{p_}.self_attn.k_proj.weight"] = rand(rng, KVH * HD, H) + w[f"{p_}.self_attn.v_proj.weight"] = rand(rng, KVH * HD, H) + w[f"{p_}.self_attn.o_proj.weight"] = rand(rng, H, HEADS * HD) + w[f"{p_}.self_attn.q_norm.weight"] = ones(HD) + w[f"{p_}.self_attn.k_norm.weight"] = ones(HD) + w[f"{p_}.input_layernorm.weight"] = ones(H) + w[f"{p_}.post_attention_layernorm.weight"] = ones(H) + w[f"{p_}.mlp.gate.weight"] = rand(rng, EXPERTS, H) + # Per-expert tensors, the layout a real checkpoint ships; sanitize stacks + # these into switch_mlp. + for e in range(EXPERTS): + w[f"{p_}.mlp.experts.{e}.gate_proj.weight"] = rand(rng, MOE_INTER, H) + w[f"{p_}.mlp.experts.{e}.up_proj.weight"] = rand(rng, MOE_INTER, H) + w[f"{p_}.mlp.experts.{e}.down_proj.weight"] = rand(rng, H, MOE_INTER) + + save_file(w, os.path.join(out, "model.safetensors"), metadata={"format": "pt"}) + return len(w) + + FIXTURES = { "dense": (build_dense, {}), "stray-shard": (build_dense, {"stray_shard": True}), "kv-shared-absent": (build_gemma4_kv_shared, {"vestigial": False}), "kv-shared-present": (build_gemma4_kv_shared, {"vestigial": True}), + "moe-nested": (build_moe_nested, {}), } if __name__ == "__main__": diff --git a/tests/fixtures/moe-nested/config.json b/tests/fixtures/moe-nested/config.json new file mode 100644 index 0000000..bfee7eb --- /dev/null +++ b/tests/fixtures/moe-nested/config.json @@ -0,0 +1,44 @@ +{ + "model_type": "qwen3_moe", + "hidden_size": 64, + "num_hidden_layers": 2, + "intermediate_size": 128, + "num_attention_heads": 4, + "num_key_value_heads": 2, + "head_dim": 16, + "num_experts": 4, + "num_experts_per_tok": 2, + "moe_intermediate_size": 32, + "decoder_sparse_step": 1, + "mlp_only_layers": [], + "rms_norm_eps": 1e-06, + "vocab_size": 288, + "rope_theta": 10000.0, + "tie_word_embeddings": false, + "max_position_embeddings": 512, + "norm_topk_prob": true, + "text_config": { + "model_type": "qwen3_moe", + "hidden_size": 64, + "num_hidden_layers": 2, + "intermediate_size": 128, + "num_attention_heads": 4, + "num_key_value_heads": 2, + "head_dim": 16, + "num_experts": 4, + "num_experts_per_tok": 2, + "moe_intermediate_size": 32, + "decoder_sparse_step": 1, + "mlp_only_layers": [], + "rms_norm_eps": 1e-06, + "vocab_size": 288, + "rope_theta": 10000.0, + "tie_word_embeddings": false, + "max_position_embeddings": 512, + "norm_topk_prob": true + }, + "vision_config": { + "model_type": "qwen3_vl", + "num_experts": 999 + } +} \ No newline at end of file diff --git a/tests/fixtures/moe-nested/model.safetensors b/tests/fixtures/moe-nested/model.safetensors new file mode 100644 index 0000000..fab58e1 Binary files /dev/null and b/tests/fixtures/moe-nested/model.safetensors differ diff --git a/tests/fixtures/moe-nested/tokenizer.json b/tests/fixtures/moe-nested/tokenizer.json new file mode 100644 index 0000000..21c0e13 --- /dev/null +++ b/tests/fixtures/moe-nested/tokenizer.json @@ -0,0 +1,380 @@ +{ + "version": "1.0", + "truncation": null, + "padding": null, + "added_tokens": [ + { + "id": 0, + "content": "<|endoftext|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1, + "content": "<|im_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2, + "content": "<|im_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + } + ], + "normalizer": null, + "pre_tokenizer": { + "type": "ByteLevel", + "add_prefix_space": false, + "trim_offsets": true, + "use_regex": true + }, + "post_processor": { + "type": "ByteLevel", + "add_prefix_space": true, + "trim_offsets": false, + "use_regex": true + }, + "decoder": { + "type": "ByteLevel", + "add_prefix_space": true, + "trim_offsets": true, + "use_regex": true + }, + "model": { + "type": "BPE", + "dropout": null, + "unk_token": null, + "continuing_subword_prefix": null, + "end_of_word_suffix": null, + "fuse_unk": false, + "byte_fallback": false, + "ignore_merges": false, + "vocab": { + "<|endoftext|>": 0, + "<|im_start|>": 1, + "<|im_end|>": 2, + "": 3, + "!": 4, + "\"": 5, + "#": 6, + "$": 7, + "%": 8, + "&": 9, + "'": 10, + "(": 11, + ")": 12, + "*": 13, + "+": 14, + ",": 15, + "-": 16, + ".": 17, + "/": 18, + "0": 19, + "1": 20, + "2": 21, + "3": 22, + "4": 23, + "5": 24, + "6": 25, + "7": 26, + "8": 27, + "9": 28, + ":": 29, + ";": 30, + "<": 31, + "=": 32, + ">": 33, + "?": 34, + "@": 35, + "A": 36, + "B": 37, + "C": 38, + "D": 39, + "E": 40, + "F": 41, + "G": 42, + "H": 43, + "I": 44, + "J": 45, + "K": 46, + "L": 47, + "M": 48, + "N": 49, + "O": 50, + "P": 51, + "Q": 52, + "R": 53, + "S": 54, + "T": 55, + "U": 56, + "V": 57, + "W": 58, + "X": 59, + "Y": 60, + "Z": 61, + "[": 62, + "\\": 63, + "]": 64, + "^": 65, + "_": 66, + "`": 67, + "a": 68, + "b": 69, + "c": 70, + "d": 71, + "e": 72, + "f": 73, + "g": 74, + "h": 75, + "i": 76, + "j": 77, + "k": 78, + "l": 79, + "m": 80, + "n": 81, + "o": 82, + "p": 83, + "q": 84, + "r": 85, + "s": 86, + "t": 87, + "u": 88, + "v": 89, + "w": 90, + "x": 91, + "y": 92, + "z": 93, + "{": 94, + "|": 95, + "}": 96, + "~": 97, + "¡": 98, + "¢": 99, + "£": 100, + "¤": 101, + "¥": 102, + "¦": 103, + "§": 104, + "¨": 105, + "©": 106, + "ª": 107, + "«": 108, + "¬": 109, + "®": 110, + "¯": 111, + "°": 112, + "±": 113, + "²": 114, + "³": 115, + "´": 116, + "µ": 117, + "¶": 118, + "·": 119, + "¸": 120, + "¹": 121, + "º": 122, + "»": 123, + "¼": 124, + "½": 125, + "¾": 126, + "¿": 127, + "À": 128, + "Á": 129, + "Â": 130, + "Ã": 131, + "Ä": 132, + "Å": 133, + "Æ": 134, + "Ç": 135, + "È": 136, + "É": 137, + "Ê": 138, + "Ë": 139, + "Ì": 140, + "Í": 141, + "Î": 142, + "Ï": 143, + "Ð": 144, + "Ñ": 145, + "Ò": 146, + "Ó": 147, + "Ô": 148, + "Õ": 149, + "Ö": 150, + "×": 151, + "Ø": 152, + "Ù": 153, + "Ú": 154, + "Û": 155, + "Ü": 156, + "Ý": 157, + "Þ": 158, + "ß": 159, + "à": 160, + "á": 161, + "â": 162, + "ã": 163, + "ä": 164, + "å": 165, + "æ": 166, + "ç": 167, + "è": 168, + "é": 169, + "ê": 170, + "ë": 171, + "ì": 172, + "í": 173, + "î": 174, + "ï": 175, + "ð": 176, + "ñ": 177, + "ò": 178, + "ó": 179, + "ô": 180, + "õ": 181, + "ö": 182, + "÷": 183, + "ø": 184, + "ù": 185, + "ú": 186, + "û": 187, + "ü": 188, + "ý": 189, + "þ": 190, + "ÿ": 191, + "Ā": 192, + "ā": 193, + "Ă": 194, + "ă": 195, + "Ą": 196, + "ą": 197, + "Ć": 198, + "ć": 199, + "Ĉ": 200, + "ĉ": 201, + "Ċ": 202, + "ċ": 203, + "Č": 204, + "č": 205, + "Ď": 206, + "ď": 207, + "Đ": 208, + "đ": 209, + "Ē": 210, + "ē": 211, + "Ĕ": 212, + "ĕ": 213, + "Ė": 214, + "ė": 215, + "Ę": 216, + "ę": 217, + "Ě": 218, + "ě": 219, + "Ĝ": 220, + "ĝ": 221, + "Ğ": 222, + "ğ": 223, + "Ġ": 224, + "ġ": 225, + "Ģ": 226, + "ģ": 227, + "Ĥ": 228, + "ĥ": 229, + "Ħ": 230, + "ħ": 231, + "Ĩ": 232, + "ĩ": 233, + "Ī": 234, + "ī": 235, + "Ĭ": 236, + "ĭ": 237, + "Į": 238, + "į": 239, + "İ": 240, + "ı": 241, + "IJ": 242, + "ij": 243, + "Ĵ": 244, + "ĵ": 245, + "Ķ": 246, + "ķ": 247, + "ĸ": 248, + "Ĺ": 249, + "ĺ": 250, + "Ļ": 251, + "ļ": 252, + "Ľ": 253, + "ľ": 254, + "Ŀ": 255, + "ŀ": 256, + "Ł": 257, + "ł": 258, + "Ń": 259, + "he": 260, + "ll": 261, + "hell": 262, + "te": 263, + "<|unused264|>": 264, + "<|unused265|>": 265, + "<|unused266|>": 266, + "<|unused267|>": 267, + "<|unused268|>": 268, + "<|unused269|>": 269, + "<|unused270|>": 270, + "<|unused271|>": 271, + "<|unused272|>": 272, + "<|unused273|>": 273, + "<|unused274|>": 274, + "<|unused275|>": 275, + "<|unused276|>": 276, + "<|unused277|>": 277, + "<|unused278|>": 278, + "<|unused279|>": 279, + "<|unused280|>": 280, + "<|unused281|>": 281, + "<|unused282|>": 282, + "<|unused283|>": 283, + "<|unused284|>": 284, + "<|unused285|>": 285, + "<|unused286|>": 286, + "<|unused287|>": 287 + }, + "merges": [ + [ + "h", + "e" + ], + [ + "l", + "l" + ], + [ + "he", + "ll" + ], + [ + "t", + "e" + ] + ] + } +} \ No newline at end of file diff --git a/tests/fixtures/moe-nested/tokenizer_config.json b/tests/fixtures/moe-nested/tokenizer_config.json new file mode 100644 index 0000000..ee42569 --- /dev/null +++ b/tests/fixtures/moe-nested/tokenizer_config.json @@ -0,0 +1,8 @@ +{ + "tokenizer_class": "PreTrainedTokenizerFast", + "bos_token": "<|endoftext|>", + "eos_token": "<|im_end|>", + "pad_token": "", + "unk_token": "<|endoftext|>", + "chat_template": "{% for m in messages %}<|im_start|>{{ m['role'] }}\n{{ m['content'] }}<|im_end|>\n{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}" +} \ No newline at end of file diff --git a/tests/test-fixtures.sh b/tests/test-fixtures.sh index 230fa3a..92f2248 100755 --- a/tests/test-fixtures.sh +++ b/tests/test-fixtures.sh @@ -10,6 +10,9 @@ # stray-shard #118: a .safetensors beside the index but absent from it # kv-shared-absent #120: gemma-4-e4b shape, shared layers ship no k/v # kv-shared-present b674: gemma-4-e2b shape, shared layers ship k/v anyway +# moe-nested MoE load path: per-expert tensors stacked into switch_mlp, +# expert count nested under text_config. Note this does not +# reproduce #112 — see scripts/make-test-fixtures.py for why. # # The output is gibberish by construction — the weights are random. A fixture passes # when the server loads it and produces *a* token, which is what exercises config @@ -65,14 +68,21 @@ run_fixture() { local body body=$(curl -sf --max-time 60 "$url/v1/chat/completions" \ -H 'Content-Type: application/json' \ - -d '{"messages":[{"role":"user","content":"ping"}],"max_tokens":4,"stream":false}' 2>/dev/null) + -d '{"messages":[{"role":"user","content":"ping"}],"max_tokens":4,"temperature":0,"stream":false}' 2>/dev/null) - # Random weights make the text meaningless, so assert on the token count instead. + # What is being proved is that the checkpoint loaded and a forward pass ran, so the + # assertion is on prefill and a well-formed response — not on how many tokens came + # back. Random weights sample randomly: an early EOS yields zero completion tokens, + # which made an earlier version of this check pass or fail run to run. temperature 0 + # removes the sampling variance; prompt_tokens is what actually evidences the prefill. if [ -n "$body" ] && echo "$body" | python3 -c ' import json, sys d = json.load(sys.stdin) -sys.exit(0 if d["usage"]["completion_tokens"] >= 1 else 1)' 2>/dev/null; then - pass "$name loaded and generated" +ok = (d["usage"]["prompt_tokens"] >= 1 + and isinstance(d["choices"][0]["message"]["content"], str) + and d["choices"][0].get("finish_reason")) +sys.exit(0 if ok else 1)' 2>/dev/null; then + pass "$name loaded, prefilled and returned a well-formed completion" else fail "$name: no completion — $(echo "$body" | head -c 120)" fi @@ -82,11 +92,37 @@ sys.exit(0 if d["usage"]["completion_tokens"] >= 1 else 1)' 2>/dev/null; then } log "Binary: $BINARY" -for name in dense stray-shard kv-shared-absent kv-shared-present; do +for name in dense stray-shard kv-shared-absent kv-shared-present moe-nested; do log "Shape: $name" run_fixture "$name" done +# The MoE fixture again, this time with --stream-experts. A model classified dense +# has the flag silently dropped and gets materialised whole, which is how #112 +# reached an OOM kill rather than an error message; asserting the flag was honoured +# is the closest end-to-end check available. +log "Shape: moe-nested (--stream-experts honoured)" +MOE_LOG="/tmp/SwiftLM-test-fixture-moe-stream.log" +"$BINARY" --model "$FIXTURE_DIR/moe-nested" --port "$PORT" --host "$HOST" \ + --stream-experts > "$MOE_LOG" 2>&1 & +SERVER_PID=$! +ready=0 +for _ in $(seq 1 60); do + curl -sf "http://$HOST:$PORT/health" >/dev/null 2>&1 && { ready=1; break; } + kill -0 "$SERVER_PID" 2>/dev/null || break + sleep 1 +done +if [ "$ready" -ne 1 ]; then + fail "moe-nested did not start with --stream-experts" +elif grep -qi "is not MoE" "$MOE_LOG"; then + fail "moe-nested was classified dense: $(grep -i 'is not MoE' "$MOE_LOG" | head -1)" +elif grep -q "SSD Expert Streaming enabled" "$MOE_LOG"; then + pass "moe-nested detected as MoE, --stream-experts honoured" +else + fail "moe-nested: no streaming confirmation in log" +fi +cleanup + log "═══════════════════════════════════════" log "Results: $PASS passed, $FAIL failed" log "═══════════════════════════════════════"