Gemma4 support: pFlash + DFlash + chunked prefill, daemon mode, server routing

.gitignore

@@ -71,3 +71,6 @@ fix-plan.md
 .env.local
 *.pem
 *.key
+
+# Sisyphus working dossier (preserve outside repo)
+.sisyphus/

.gitmodules

@@ -1,7 +1,7 @@
 [submodule "dflash/deps/llama.cpp"]
 	path = dflash/deps/llama.cpp
-	url = https://github.com/Luce-Org/llama.cpp-dflash-ggml.git
-	branch = luce-dflash
+	url = https://github.com/dusterbloom/llama-cpp-turboquant-cuda.git
+	branch = feature/tq3-kv-cache-clean
 [submodule "dflash/deps/Block-Sparse-Attention"]
 	path = dflash/deps/Block-Sparse-Attention
 	url = https://github.com/mit-han-lab/Block-Sparse-Attention.git

dflash/CMakeLists.txt

@@ -205,6 +205,10 @@ add_library(dflash27b STATIC
     src/qwen3/qwen3_drafter.cpp
     src/qwen3/qwen3_loader.cpp
     src/qwen3/qwen3_graph.cpp
+    src/gemma4_target_loader.cpp
+    src/gemma4_target_graph.cpp
+    src/gemma4_mtp_graph.cpp
+    src/gemma4_dflash_graph.cpp
     src/flashprefill_q8.cpp
     src/kv_cache.cpp
     src/kv_quant.cpp
@@ -246,6 +250,11 @@ elseif(DFLASH27B_GPU_BACKEND STREQUAL "hip")
     target_compile_definitions(dflash27b PRIVATE DFLASH27B_BACKEND_HIP=1 GGML_USE_HIP)
 endif()
 
+# Backward-compat alias for our gemma4 graph code that uses DFLASH27B_MIN_SM.
+# origin/main renamed the variable to _dflash27b_cuda_min_sm; expose both names
+# so dflash/src/gemma4_dflash_graph.cpp keeps building unchanged.
+target_compile_definitions(dflash27b PRIVATE DFLASH27B_MIN_SM=${_dflash27b_cuda_min_sm})
+
 # FlashPrefill custom kernels.
 #   CUDA: BF16 WMMA needs sm_80+; on sm_75 we fall back to ggml flash_attn_ext.
 #   HIP Phase 1 (default): ggml q8 fallback, no custom kernels.
@@ -283,7 +292,8 @@ elseif(DFLASH27B_GPU_BACKEND STREQUAL "cuda" AND _dflash27b_cuda_min_sm GREATER_
     target_sources(dflash27b PRIVATE
         src/flashprefill_kernels.cu
         src/flashprefill_select.cpp
-        src/flashprefill.cpp)
+        src/flashprefill.cpp
+        src/pflash_ggml_adapter.cpp)
     target_compile_definitions(dflash27b PRIVATE DFLASH27B_HAVE_CUDA_WMMA_FLASHPREFILL=1)
 endif()
 
@@ -525,5 +535,78 @@ if(DFLASH27B_TESTS)
             target_link_libraries(${_t} PRIVATE CUDA::cudart)
         endif()
     endforeach()
+
+    if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/gemma4/test_gemma4_dflash.cpp")
+        add_executable(test_gemma4_dflash test/gemma4/test_gemma4_dflash.cpp)
+        target_include_directories(test_gemma4_dflash PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src)
+        target_link_libraries(test_gemma4_dflash PRIVATE dflash27b ggml ggml-cuda)
+        find_package(CUDAToolkit REQUIRED)
+        target_link_libraries(test_gemma4_dflash PRIVATE CUDA::cudart)
+    endif()
+
+    if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/gemma4/smoke_load_gemma4_target.cpp")
+        add_executable(smoke_load_gemma4_target test/gemma4/smoke_load_gemma4_target.cpp)
+        target_include_directories(smoke_load_gemma4_target PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src)
+        target_link_libraries(smoke_load_gemma4_target PRIVATE dflash27b ggml ggml-cuda)
+        find_package(CUDAToolkit REQUIRED)
+        target_link_libraries(smoke_load_gemma4_target PRIVATE CUDA::cudart)
+    endif()
+
+    if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/gemma4/smoke_gemma4_target_forward.cpp")
+        add_executable(smoke_gemma4_target_forward test/gemma4/smoke_gemma4_target_forward.cpp)
+        target_include_directories(smoke_gemma4_target_forward PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src)
+        target_link_libraries(smoke_gemma4_target_forward PRIVATE dflash27b ggml ggml-cuda)
+        find_package(CUDAToolkit REQUIRED)
+        target_link_libraries(smoke_gemma4_target_forward PRIVATE CUDA::cudart)
+    endif()
+
+    if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/gemma4/smoke_load_gemma4_draft.cpp")
+        add_executable(smoke_load_gemma4_draft test/gemma4/smoke_load_gemma4_draft.cpp)
+        target_include_directories(smoke_load_gemma4_draft PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src)
+        target_link_libraries(smoke_load_gemma4_draft PRIVATE dflash27b ggml ggml-cuda)
+        find_package(CUDAToolkit REQUIRED)
+        target_link_libraries(smoke_load_gemma4_draft PRIVATE CUDA::cudart)
+    endif()
+
+    if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/gemma4/smoke_gemma4_draft_forward.cpp")
+        add_executable(smoke_gemma4_draft_forward test/gemma4/smoke_gemma4_draft_forward.cpp)
+        target_include_directories(smoke_gemma4_draft_forward PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src)
+        target_link_libraries(smoke_gemma4_draft_forward PRIVATE dflash27b ggml ggml-cuda)
+        find_package(CUDAToolkit REQUIRED)
+        target_link_libraries(smoke_gemma4_draft_forward PRIVATE CUDA::cudart)
+    endif()
+
+    if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/gemma4/test_gemma4_kv_tq3.cpp")
+        add_executable(test_gemma4_kv_tq3 test/gemma4/test_gemma4_kv_tq3.cpp)
+        target_include_directories(test_gemma4_kv_tq3 PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src)
+        target_link_libraries(test_gemma4_kv_tq3 PRIVATE dflash27b ggml ggml-cuda)
+        find_package(CUDAToolkit REQUIRED)
+        target_link_libraries(test_gemma4_kv_tq3 PRIVATE CUDA::cudart)
+    endif()
+
+    if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/test_flash_attn_sparse.cpp")
+        add_executable(test_flash_attn_sparse test/test_flash_attn_sparse.cpp)
+        target_link_libraries(test_flash_attn_sparse PRIVATE dflash27b ggml ggml-cuda ggml-base)
+        target_include_directories(test_flash_attn_sparse PRIVATE
+            ${CMAKE_CURRENT_SOURCE_DIR}/deps/llama.cpp/ggml/include
+            ${CMAKE_CURRENT_SOURCE_DIR}/deps/llama.cpp/ggml/src
+            ${CMAKE_CURRENT_SOURCE_DIR}/src)
+    endif()
+
+    if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/gemma4/test_mtp_loader.cpp")
+        add_executable(test_mtp_loader test/gemma4/test_mtp_loader.cpp)
+        target_include_directories(test_mtp_loader PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src)
+        target_link_libraries(test_mtp_loader PRIVATE dflash27b ggml ggml-cuda)
+        find_package(CUDAToolkit REQUIRED)
+        target_link_libraries(test_mtp_loader PRIVATE CUDA::cudart)
+    endif()
+
+    if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/gemma4/test_mtp_graph_shapes.cpp")
+        add_executable(test_mtp_graph_shapes test/gemma4/test_mtp_graph_shapes.cpp)
+        target_include_directories(test_mtp_graph_shapes PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src)
+        target_link_libraries(test_mtp_graph_shapes PRIVATE dflash27b ggml ggml-cuda)
+        find_package(CUDAToolkit REQUIRED)
+        target_link_libraries(test_mtp_graph_shapes PRIVATE CUDA::cudart)
     endif()
+    endif()  # DFLASH27B_GPU_BACKEND STREQUAL "cuda"
 endif()

dflash/README.md

@@ -328,6 +328,14 @@ DFLASH27B_KV_TQ3=1 DFLASH27B_PREFILL_UBATCH=16 \
 
 **Requirements:** NVIDIA sm_75+ GPU (2080 Ti, 3090, A10, A40, 4090) or Jetson AGX Thor sm_110, CUDA 12+ (CUDA 13+ required for Thor), 22+ GB VRAM, ~80 GB disk. On Turing (SM 7.5), BF16 draft weights are auto-converted to FP16 at load time for tensor core acceleration.
 
+### Small-VRAM cards (<=24 GiB)
+
+VMM-backed pools waste VRAM on cards under ~24 GiB. The 32 GB VMM pool reservation fragments badly on a 24 GB card and causes prefill+verify cliffs (measured ~50% throughput loss at ctx=64K). Build with:
+
+    cmake -DGGML_CUDA_NO_VMM=ON ..
+
+`GGML_CUDA_NO_VMM` is a **compile-time** CMake option — it cannot be set at runtime via environment variable. The dflash test binary prints a runtime warning if it detects <=24 GiB VRAM and the binary was built without this flag.
+
 ## How it works
 
 **Block-diffusion draft.** Each step, the draft sees `[last_target_token, MASK×15]` plus the last 5 captured target hidden states. It denoises the masks in a single forward, producing 16 candidate tokens conditioned on real target features. Structurally stronger than chain EAGLE: every position conditions on the same captured context, not its own noisy predictions.

dflash/include/gemma4.h

@@ -0,0 +1,62 @@
+// gemma4 — standalone CUDA library for DFlash speculative decoding of
+// Gemma4 models (31B Dense and 26B-A4B MoE) with a DFlash draft model.
+
+#ifndef GEMMA4_H
+#define GEMMA4_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// ─── Gemma4-31B Dense config ───────────────────────────────────────
+
+#define GEMMA4_31B_HIDDEN              4096
+#define GEMMA4_31B_LAYERS              60
+#define GEMMA4_31B_N_HEADS             32
+#define GEMMA4_31B_N_KV_HEADS          8
+#define GEMMA4_31B_HEAD_DIM            128
+#define GEMMA4_31B_INTERMEDIATE        16384
+#define GEMMA4_31B_VOCAB               262144
+#define GEMMA4_31B_SWA_WINDOW          1024
+
+// ─── Gemma4-26B-A4B MoE config ────────────────────────────────────
+
+#define GEMMA4_26B_HIDDEN              4096
+#define GEMMA4_26B_LAYERS              30
+#define GEMMA4_26B_N_HEADS             32
+#define GEMMA4_26B_N_KV_HEADS          8
+#define GEMMA4_26B_HEAD_DIM            128
+#define GEMMA4_26B_INTERMEDIATE        16384
+#define GEMMA4_26B_EXPERT_INTERMEDIATE 2048
+#define GEMMA4_26B_N_EXPERTS           128
+#define GEMMA4_26B_N_EXPERTS_USED      8
+#define GEMMA4_26B_VOCAB               262144
+#define GEMMA4_26B_SWA_WINDOW          1024
+
+// ─── Shared constants ─────────────────────────────────────────────
+
+#define GEMMA4_ROPE_THETA              1000000.0f
+#define GEMMA4_RMS_EPS                 1e-6f
+#define GEMMA4_LOGIT_SOFTCAP           30.0f
+#define GEMMA4_ATTN_SCALE              1.0f
+
+// ─── Draft model config ───────────────────────────────────────────
+
+#define GEMMA4_DRAFT_LAYERS            5
+#define GEMMA4_DRAFT_BLOCK_SIZE        16
+#define GEMMA4_DRAFT_N_TARGET_LAYERS   6
+#define GEMMA4_31B_DRAFT_MASK_TOKEN_ID 4
+#define GEMMA4_26B_DRAFT_MASK_TOKEN_ID 4
+
+// ─── Diagnostics ──────────────────────────────────────────────────
+
+const char * gemma4_last_error(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // GEMMA4_H