From 46a39ed2a352cfaeeaa57cab13ecb703b88b34c7 Mon Sep 17 00:00:00 2001 From: Pascal Date: Wed, 16 Sep 2026 16:19:16 +0200 Subject: [PATCH 1/2] llama: read the SWA pattern as a period or a per-layer array Add llama_model_base::load_swa_pattern(), which reads sliding_window_pattern either as one flag per layer or as a period expanded by set_swa_pattern(), and use it in every loader that reads the key as a period. These loaders silently ignored an array and applied their default period, although the converters of olmo2, gemma3n and exaone4 write arrays. The published GGUFs match the defaults, so their outputs do not change. The loaders that already accepted both forms lose their duplicated scalar-then-array block, and use their declared default period when the key is absent. --- src/llama-model.cpp | 9 +++++++++ src/llama-model.h | 3 +++ src/models/afmoe.cpp | 4 +--- src/models/cohere2.cpp | 4 +--- src/models/cohere2moe.cpp | 7 +------ src/models/exaone-moe.cpp | 4 +--- src/models/exaone4.cpp | 4 +--- src/models/gemma-embedding.cpp | 4 +--- src/models/gemma2.cpp | 4 +--- src/models/gemma3.cpp | 4 +--- src/models/gemma3n.cpp | 4 +--- src/models/laguna.cpp | 4 +--- src/models/llama4.cpp | 4 +--- src/models/mellum.cpp | 8 +------- src/models/modern-bert.cpp | 4 +--- src/models/muse-glimmer.cpp | 7 +------ src/models/olmo2.cpp | 4 +--- src/models/openai-moe.cpp | 4 +--- src/models/plamo3.cpp | 4 +--- src/models/smallthinker.cpp | 4 +--- 20 files changed, 30 insertions(+), 64 deletions(-) diff --git a/src/llama-model.cpp b/src/llama-model.cpp index de3b2e38ff62..e195f50d0bde 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -3305,6 +3305,15 @@ void llama_model_base::create_tensor_qkv(llama_layer & layer, int bid, } } +void llama_model_base::load_swa_pattern(llama_model_loader & ml, uint32_t n_pattern, bool dense_first) { + if (ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, false)) { + return; + } + + ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, n_pattern, false); + hparams.set_swa_pattern(n_pattern, dense_first); +} + const int32_t * llama_model_target_layer_ids(const struct llama_model * model) { const auto & v = model->target_layer_ids; return v.empty() ? nullptr : v.data(); diff --git a/src/llama-model.h b/src/llama-model.h index d61afa2cb36f..984b2cf38837 100644 --- a/src/llama-model.h +++ b/src/llama-model.h @@ -814,6 +814,9 @@ struct llama_model_base : public llama_model { int64_t n_embd_, int64_t n_embd_q_, int64_t n_embd_k_, int64_t n_embd_v_, int flags); + // helper: read the SWA pattern as one flag per layer, or as a period expanded by set_swa_pattern + void load_swa_pattern(llama_model_loader & ml, uint32_t n_pattern, bool dense_first = false); + void load_stats (llama_model_loader & ml) override; void load_hparams(llama_model_loader & ml) override; void load_vocab (llama_model_loader & ml) override; diff --git a/src/models/afmoe.cpp b/src/models/afmoe.cpp index cf0220367186..08c22b6ed823 100644 --- a/src/models/afmoe.cpp +++ b/src/models/afmoe.cpp @@ -14,9 +14,7 @@ void llama_model_afmoe::load_arch_hparams(llama_model_loader & ml) { // Pattern: 3 sliding - 1 full (global_attn_every_n_layers = 4) if (hparams.n_swa > 0) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; - uint32_t swa_period = 4; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period); + load_swa_pattern(ml, 4); hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; diff --git a/src/models/cohere2.cpp b/src/models/cohere2.cpp index e2b3662560df..7ad5244e427c 100644 --- a/src/models/cohere2.cpp +++ b/src/models/cohere2.cpp @@ -2,9 +2,7 @@ void llama_model_cohere2::load_arch_hparams(llama_model_loader & ml) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; - uint32_t swa_period = 4; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period); + load_swa_pattern(ml, 4); hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; diff --git a/src/models/cohere2moe.cpp b/src/models/cohere2moe.cpp index 5e02cd56e71d..7704cbb87299 100644 --- a/src/models/cohere2moe.cpp +++ b/src/models/cohere2moe.cpp @@ -25,12 +25,7 @@ void llama_model_cohere2moe::load_arch_hparams(llama_model_loader & ml) { } hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; - uint32_t swa_period = 4; - if (ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false)) { - hparams.set_swa_pattern(swa_period, true); - } else { - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, hparams.n_layer()); - } + load_swa_pattern(ml, 4, true); hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; diff --git a/src/models/exaone-moe.cpp b/src/models/exaone-moe.cpp index 976ee050adcd..840da5f67875 100644 --- a/src/models/exaone-moe.cpp +++ b/src/models/exaone-moe.cpp @@ -3,9 +3,7 @@ void llama_model_exaone_moe::load_arch_hparams(llama_model_loader & ml) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; hparams.n_swa = 128; - uint32_t swa_period = 4; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period); + load_swa_pattern(ml, 4); hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; diff --git a/src/models/exaone4.cpp b/src/models/exaone4.cpp index 9ba978956dc8..c1c55957a4fd 100644 --- a/src/models/exaone4.cpp +++ b/src/models/exaone4.cpp @@ -4,9 +4,7 @@ void llama_model_exaone4::load_arch_hparams(llama_model_loader & ml) { if (hparams.n_layer() == 64) { // 32B hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; hparams.n_swa = 4096; - uint32_t swa_period = 4; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period); + load_swa_pattern(ml, 4); hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; diff --git a/src/models/gemma-embedding.cpp b/src/models/gemma-embedding.cpp index 80ed3b1a4605..6c97883d8279 100644 --- a/src/models/gemma-embedding.cpp +++ b/src/models/gemma-embedding.cpp @@ -2,9 +2,7 @@ void llama_model_gemma_embedding::load_arch_hparams(llama_model_loader & ml) { hparams.swa_type = LLAMA_SWA_TYPE_SYMMETRIC; - uint32_t swa_period = 6; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period); + load_swa_pattern(ml, 6); hparams.causal_attn = false; // embeddings do not use causal attention diff --git a/src/models/gemma2.cpp b/src/models/gemma2.cpp index 2fbfb15a94a3..9e5243465543 100644 --- a/src/models/gemma2.cpp +++ b/src/models/gemma2.cpp @@ -3,9 +3,7 @@ void llama_model_gemma2::load_arch_hparams(llama_model_loader & ml) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; hparams.n_swa = 4096; // default value of gemma 2 - uint32_t swa_period = 2; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period); + load_swa_pattern(ml, 2); hparams.attn_soft_cap = true; hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; diff --git a/src/models/gemma3.cpp b/src/models/gemma3.cpp index 690194529e38..f99bbaacd8ad 100644 --- a/src/models/gemma3.cpp +++ b/src/models/gemma3.cpp @@ -4,9 +4,7 @@ void llama_model_gemma3::load_arch_hparams(llama_model_loader & ml) { const bool found_swa = ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa, false); if (found_swa && hparams.n_swa > 0) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; - uint32_t swa_period = 6; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period); + load_swa_pattern(ml, 6); ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false); } else { diff --git a/src/models/gemma3n.cpp b/src/models/gemma3n.cpp index bb628203aaaf..4d47ddc62fc9 100644 --- a/src/models/gemma3n.cpp +++ b/src/models/gemma3n.cpp @@ -1,10 +1,8 @@ #include "models.h" void llama_model_gemma3n::load_arch_hparams(llama_model_loader & ml) { - uint32_t swa_period = 5; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; - hparams.set_swa_pattern(swa_period); + load_swa_pattern(ml, 5); hparams.n_layer_kv_from_start = 20; hparams.f_attention_scale = 1.0f; diff --git a/src/models/laguna.cpp b/src/models/laguna.cpp index 556400bfcef1..2bc4fa8a0fc3 100644 --- a/src/models/laguna.cpp +++ b/src/models/laguna.cpp @@ -36,9 +36,7 @@ void llama_model_laguna::load_arch_hparams(llama_model_loader & ml) { if (hparams.n_swa > 0) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; - uint32_t swa_period = 4; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period, /*dense_first=*/true); // XS.2: FULL at il%4==0 + load_swa_pattern(ml, 4, /*dense_first=*/true); // XS.2: FULL at il%4==0 // Per-layer-type RoPE: full layers use YaRN θ=500000 over 64 dims; // SWA layers use default RoPE θ=10000 over 128 dims. Base load_hparams diff --git a/src/models/llama4.cpp b/src/models/llama4.cpp index 8a812beffac4..4f79b4d1eefe 100644 --- a/src/models/llama4.cpp +++ b/src/models/llama4.cpp @@ -16,9 +16,7 @@ void llama_model_llama4::load_arch_hparams(llama_model_loader & ml) { hparams.f_attn_temp_scale = 0.1f; hparams.f_attn_temp_offset = 1.0f; - uint32_t swa_period = 4; // pattern: 3 chunked - 1 full - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period); + load_swa_pattern(ml, 4); // pattern: 3 chunked - 1 full hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; diff --git a/src/models/mellum.cpp b/src/models/mellum.cpp index 872a9c8f556f..3ab79747af27 100644 --- a/src/models/mellum.cpp +++ b/src/models/mellum.cpp @@ -8,13 +8,7 @@ void llama_model_mellum::load_arch_hparams(llama_model_loader & ml) { if (hparams.n_swa > 0) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; - uint32_t swa_period = 4; - const auto res = ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - if (res) { - hparams.set_swa_pattern(swa_period); - } else { - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, hparams.n_layer()); - } + load_swa_pattern(ml, 4); hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; diff --git a/src/models/modern-bert.cpp b/src/models/modern-bert.cpp index f3e9407e0125..b7542d59bd54 100644 --- a/src/models/modern-bert.cpp +++ b/src/models/modern-bert.cpp @@ -5,9 +5,7 @@ void llama_model_modern_bert::load_arch_hparams(llama_model_loader & ml) { if (found_swa && hparams.n_swa > 0) { hparams.swa_type = LLAMA_SWA_TYPE_SYMMETRIC; ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false); - uint32_t swa_period = 3; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period, true); + load_swa_pattern(ml, 3, true); } else { hparams.swa_type = LLAMA_SWA_TYPE_NONE; } diff --git a/src/models/muse-glimmer.cpp b/src/models/muse-glimmer.cpp index 0e94153088a9..0e5f75ebed9e 100644 --- a/src/models/muse-glimmer.cpp +++ b/src/models/muse-glimmer.cpp @@ -10,12 +10,7 @@ void llama_model_muse_glimmer::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false); hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; - uint32_t swa_period = 4; - if (ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false)) { - hparams.set_swa_pattern(swa_period); - } else { - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, hparams.n_layer()); - } + load_swa_pattern(ml, 4); switch (hparams.n_layer()) { case 52: type = LLM_TYPE_30B; break; diff --git a/src/models/olmo2.cpp b/src/models/olmo2.cpp index 05b9394b8fe4..fe5dc88d4469 100644 --- a/src/models/olmo2.cpp +++ b/src/models/olmo2.cpp @@ -6,9 +6,7 @@ void llama_model_olmo2::load_arch_hparams(llama_model_loader & ml) { const bool found_swa = ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa, false); if (found_swa && hparams.n_swa > 0) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; - uint32_t swa_period = 4; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period); + load_swa_pattern(ml, 4); hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = 1.0; // See olmo2.cpp diff --git a/src/models/openai-moe.cpp b/src/models/openai-moe.cpp index c9f9b677d06e..39660a325dd4 100644 --- a/src/models/openai-moe.cpp +++ b/src/models/openai-moe.cpp @@ -6,9 +6,7 @@ void llama_model_openai_moe::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; - uint32_t swa_period = 2; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period); + load_swa_pattern(ml, 2); hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; diff --git a/src/models/plamo3.cpp b/src/models/plamo3.cpp index 16d0b1dcef71..f8235f8a8323 100644 --- a/src/models/plamo3.cpp +++ b/src/models/plamo3.cpp @@ -6,9 +6,7 @@ void llama_model_plamo3::load_arch_hparams(llama_model_loader & ml) { if (found_swa && hparams.n_swa > 0) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false); - uint32_t swa_period = 8; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period); + load_swa_pattern(ml, 8); } else { hparams.swa_type = LLAMA_SWA_TYPE_NONE; } diff --git a/src/models/smallthinker.cpp b/src/models/smallthinker.cpp index 680ffb8fda37..555f8b718470 100644 --- a/src/models/smallthinker.cpp +++ b/src/models/smallthinker.cpp @@ -6,9 +6,7 @@ void llama_model_smallthinker::load_arch_hparams(llama_model_loader & ml) { if (found_swa && hparams.n_swa > 0) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; hparams.n_swa = 4096; - uint32_t swa_period = 4; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period, true); + load_swa_pattern(ml, 4, true); hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; From 6e29ae1ee476c9894592a36f62cec0c91bbad992 Mon Sep 17 00:00:00 2001 From: Pascal Date: Wed, 16 Sep 2026 16:19:31 +0200 Subject: [PATCH 2/2] model-saver: write the SWA pattern and the MLA SWA geometry Write sliding_window_pattern as one flag per layer, nextn layers included, for every model using SWA. The array is never collapsed to a scalar, since the loaders read a scalar as a period. Also write the MLA key/value lengths and KV LoRA rank of the SWA layers, required by dots3note. This enables the saver for plamo3, gemma3, cohere2, cohere2moe, olmo2, exaone-moe, afmoe, mimo2, spark2_5, muse-glimmer, mellum, laguna, granite_swa, dots3note and maple, all passing the bit-exact roundtrip of test-llama-archs. --- src/llama-model-saver.cpp | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp index 0a27367c9830..0f5155b2e46f 100644 --- a/src/llama-model-saver.cpp +++ b/src/llama-model-saver.cpp @@ -15,26 +15,11 @@ bool llama_model_saver_supports_arch(llm_arch arch) { switch (arch) { - case LLM_ARCH_PLAMO3: - case LLM_ARCH_GEMMA3: case LLM_ARCH_GEMMA3N: - case LLM_ARCH_COHERE2: - case LLM_ARCH_COHERE2MOE: - case LLM_ARCH_OLMO2: case LLM_ARCH_BITNET: case LLM_ARCH_T5: - case LLM_ARCH_EXAONE_MOE: - case LLM_ARCH_AFMOE: case LLM_ARCH_APERTUS: - case LLM_ARCH_MIMO2: case LLM_ARCH_STEP35: - case LLM_ARCH_SPARK2_5: - case LLM_ARCH_MUSE_GLIMMER: - case LLM_ARCH_MELLUM: - case LLM_ARCH_LAGUNA: - case LLM_ARCH_GRANITE_SWA: - case LLM_ARCH_DOTS3NOTE: // TODO: need to handle SWA pattern and MLA+SWA config - case LLM_ARCH_MAPLE: return false; default: return true; @@ -290,7 +275,11 @@ void llama_model_saver::add_kv_from_model() { add_kv(LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT, hparams.n_rel_attn_bkts); add_kv(LLM_KV_ATTENTION_ROPE_PATTERN, hparams.rope_pattern, true); add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); - // add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, ???); + if (hparams.swa_type != LLAMA_SWA_TYPE_NONE) { + // never collapsed to a scalar: the loaders read a scalar as a period + add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, std::vector( + hparams.is_swa_impl.begin(), hparams.is_swa_impl.begin() + hparams.n_layer_all)); + } add_kv(LLM_KV_ATTENTION_SCALE, hparams.f_attention_scale); add_kv(LLM_KV_ATTENTION_OUTPUT_SCALE, hparams.f_attn_out_scale); add_kv(LLM_KV_ATTENTION_VALUE_SCALE, hparams.f_attn_value_scale); @@ -300,6 +289,9 @@ void llama_model_saver::add_kv_from_model() { add_kv(LLM_KV_ATTENTION_VALUE_LENGTH_MLA, hparams.n_embd_head_v_mla_impl); add_kv(LLM_KV_ATTENTION_KEY_LENGTH_SWA, hparams.n_embd_head_k_swa); add_kv(LLM_KV_ATTENTION_VALUE_LENGTH_SWA, hparams.n_embd_head_v_swa); + add_kv(LLM_KV_ATTENTION_KEY_LENGTH_MLA_SWA, hparams.n_embd_head_k_mla_swa); + add_kv(LLM_KV_ATTENTION_VALUE_LENGTH_MLA_SWA, hparams.n_embd_head_v_mla_swa); + add_kv(LLM_KV_ATTENTION_KV_LORA_RANK_SWA, hparams.n_lora_kv_swa); add_kv(LLM_KV_ATTENTION_INDEXER_HEAD_COUNT, hparams.indexer_n_head); add_kv(LLM_KV_ATTENTION_INDEXER_KEY_LENGTH, hparams.indexer_head_size); add_kv(LLM_KV_ATTENTION_INDEXER_TOP_K, hparams.indexer_top_k);