From fb4fca6625e9209c54f518607292f877e1f2a88c Mon Sep 17 00:00:00 2001 From: rounakbende10 Date: Thu, 3 Sep 2026 11:08:33 -0400 Subject: [PATCH] Add model configs: GLM, DeepSeek V4, Inkling, Laguna, Laguna NVFP4 - GLM 5.2 FP8: 262K context, v0.24.0 - DeepSeek V4 Flash: 1M context - DeepSeek V4 Flash NVFP4: 1M context, native FP4 on B200 - Inkling Small BF16: 1M context - Laguna S 2.1 BF16: 1M context, 30.61x concurrency on 8x H200 - Laguna S 2.1 NVFP4: 1M context, 2.33x concurrency on 1x B200 - Qwen 3.6 27B: updated max-model-len 131K to 262K --- src/coding_agent_bench/models/__init__.py | 12 +++ src/coding_agent_bench/models/configs.py | 115 +++++++++++++++++++++- 2 files changed, 126 insertions(+), 1 deletion(-) diff --git a/src/coding_agent_bench/models/__init__.py b/src/coding_agent_bench/models/__init__.py index c5a4a8c..a66588d 100644 --- a/src/coding_agent_bench/models/__init__.py +++ b/src/coding_agent_bench/models/__init__.py @@ -6,6 +6,12 @@ RedHatAI_Mistral_Small_4_119B_2603_NVFP4, RedHatAI_NVIDIA_Nemotron_3_Super_120B_A12B_NVFP4, RedHatAI_Qwen3_6_27B_FP8, + RedHatAI_GLM_5_2_FP8, + RedHatAI_DeepSeek_V4_Flash, + RedHatAI_DeepSeek_V4_Flash_NVFP4_FP8, + RedHatAI_Inkling_Small, + RedHatAI_Laguna_S_2_1, + poolside_Laguna_S_2_1_NVFP4, ) MODEL_CONFIGS: list[type[ModelConfig]] = [ @@ -15,6 +21,12 @@ RedHatAI_Mistral_Small_4_119B_2603_NVFP4, RedHatAI_NVIDIA_Nemotron_3_Super_120B_A12B_NVFP4, RedHatAI_Qwen3_6_27B_FP8, + RedHatAI_GLM_5_2_FP8, + RedHatAI_DeepSeek_V4_Flash, + RedHatAI_DeepSeek_V4_Flash_NVFP4_FP8, + RedHatAI_Inkling_Small, + RedHatAI_Laguna_S_2_1, + poolside_Laguna_S_2_1_NVFP4, ] MODEL_REGISTRY: dict[str, ModelConfig] = {cls.name: cls() for cls in MODEL_CONFIGS} diff --git a/src/coding_agent_bench/models/configs.py b/src/coding_agent_bench/models/configs.py index 7e4a5bd..255254b 100644 --- a/src/coding_agent_bench/models/configs.py +++ b/src/coding_agent_bench/models/configs.py @@ -76,7 +76,7 @@ class RedHatAI_NVIDIA_Nemotron_3_Super_120B_A12B_NVFP4(ModelConfig): ] class RedHatAI_Qwen3_6_27B_FP8(ModelConfig): - + name = "RedHatAI/Qwen3.6-27B-FP8" model_max_len = 131072 args = [ @@ -90,3 +90,116 @@ class RedHatAI_Qwen3_6_27B_FP8(ModelConfig): "--tool-call-parser", "qwen3_coder", "--default-chat-template-kwargs", '{"enable_thinking": true}', ] + + +class RedHatAI_GLM_5_2_FP8(ModelConfig): + # Verified: 8x H200 141GB, concurrency 2.23x at 262K context + # Note: cannot fit 1M context on 8x H200 (needs 52.68 GiB KV, only 23.78 GiB available) + + name = "RedHatAI/GLM-5.2-FP8" + model_max_len = 262144 + args = [ + "--model", "RedHatAI/GLM-5.2-FP8", + "--dtype", "auto", + "--max-model-len", "262144", + "--trust-remote-code", + "--kv-cache-dtype", "fp8", + "--enable-expert-parallel", + "--enable-auto-tool-choice", + "--reasoning-parser", "glm45", + "--tool-call-parser", "glm47", + ] + + +class RedHatAI_DeepSeek_V4_Flash(ModelConfig): + # Verified: 8x H200 141GB, concurrency 9.91x at 1M context + # Note: --moe-backend deep_gemm_mega_moe is B200-only (SM100) + + name = "RedHatAI/DeepSeek-V4-Flash" + image = "vllm/vllm-openai:v0.27.1" + model_max_len = 1048576 + args = [ + "--model", "RedHatAI/DeepSeek-V4-Flash", + "--dtype", "auto", + "--trust-remote-code", + "--kv-cache-dtype", "fp8", + "--block-size", "256", + "--enable-expert-parallel", + "--enable-auto-tool-choice", + "--tokenizer-mode", "deepseek_v4", + "--tool-call-parser", "deepseek_v4", + "--reasoning-parser", "deepseek_v4", + ] + + +class RedHatAI_DeepSeek_V4_Flash_NVFP4_FP8(ModelConfig): + # Verified: 8x H200 141GB, concurrency 9.81x at 1M context + # Note: Marlin FP4 fallback on H200 (no native SM100 FP4), similar concurrency to base + + name = "RedHatAI/DeepSeek-V4-Flash-NVFP4-FP8" + image = "vllm/vllm-openai:v0.27.1" + model_max_len = 1048576 + args = [ + "--model", "RedHatAI/DeepSeek-V4-Flash-NVFP4-FP8", + "--dtype", "auto", + "--trust-remote-code", + "--kv-cache-dtype", "fp8", + "--block-size", "256", + "--enable-expert-parallel", + "--enable-auto-tool-choice", + "--tokenizer-mode", "deepseek_v4", + "--tool-call-parser", "deepseek_v4", + "--reasoning-parser", "deepseek_v4", + ] + + +class RedHatAI_Inkling_Small(ModelConfig): + # Verified: 8x H200 141GB, BF16, concurrency 13.22x at 1M context + + name = "RedHatAI/Inkling-Small" + image = "vllm/vllm-openai:v0.27.1" + model_max_len = 1048576 + args = [ + "--model", "RedHatAI/Inkling-Small", + "--dtype", "auto", + "--trust-remote-code", + "--kv-cache-dtype", "fp8", + "--enable-auto-tool-choice", + "--tool-call-parser", "inkling", + "--reasoning-parser", "inkling", + ] + + +class RedHatAI_Laguna_S_2_1(ModelConfig): + # Verified: 8x H200 141GB, BF16, max-model-len 1048576, concurrency 30.61x + + name = "RedHatAI/Laguna-S-2.1" + model_max_len = 1048576 + args = [ + "--model", "RedHatAI/Laguna-S-2.1", + "--dtype", "auto", + "--trust-remote-code", + "--kv-cache-dtype", "fp8", + "--enable-auto-tool-choice", + "--reasoning-parser", "poolside_v1", + "--tool-call-parser", "poolside_v1", + "--default-chat-template-kwargs", '{"enable_thinking": true}', + ] + +class poolside_Laguna_S_2_1_NVFP4(ModelConfig): + # Verified: 1x B200 183GB, NVFP4, max-model-len 1048576, concurrency 2.33x + + name = "poolside/Laguna-S-2.1-NVFP4" + model_max_len = 1048576 + args = [ + "--model", "poolside/Laguna-S-2.1-NVFP4", + "--dtype", "auto", + "--trust-remote-code", + "--kv-cache-dtype", "fp8", + "--enable-auto-tool-choice", + "--reasoning-parser", "poolside_v1", + "--tool-call-parser", "poolside_v1", + "--default-chat-template-kwargs", '{"enable_thinking": true}', + ] + +